From b85a4fffe7a1f7c7bd18af6fb2750e1ef2ba4a5c Mon Sep 17 00:00:00 2001 From: Luis Martinez Date: Fri, 31 Dec 2021 00:16:39 -0600 Subject: [PATCH] update to 3.8.0 * add signature file for verification * package cleanup --- .SRCINFO | 14 +- CHANGELOG | 1747 +++++++++++++++++++++++++++++++++++++++++++++++++++++ PKGBUILD | 61 +- 3 files changed, 1784 insertions(+), 38 deletions(-) create mode 100644 CHANGELOG diff --git a/.SRCINFO b/.SRCINFO index 482c3f77e2c46..a4bcfa4acef9b 100644 --- a/.SRCINFO +++ b/.SRCINFO @@ -1,14 +1,12 @@ pkgbase = s3ql pkgdesc = A full-featured file system for online data storage. - pkgver = 3.7.1 + pkgver = 3.8.0 pkgrel = 1 url = https://github.com/s3ql/s3ql/ install = s3ql.install - arch = i686 + changelog = CHANGELOG arch = x86_64 license = GPL3 - makedepends = gcc - depends = python depends = python-cryptography depends = python-defusedxml depends = python-apsw @@ -19,8 +17,10 @@ pkgbase = s3ql depends = python-requests depends = python-google-auth depends = python-google-auth-oauthlib - source = https://github.com/s3ql/s3ql/releases/download/release-3.7.1/s3ql-3.7.1.tar.bz2 - md5sums = ef7062f834dd8ee0614b78cb6382e9a0 + source = s3ql-3.8.0.tar.gz::https://github.com/s3ql/s3ql//releases/download/release-3.8.0/s3ql-3.8.0.tar.gz + source = s3ql-3.8.0.tar.gz.asc::https://github.com/s3ql/s3ql//releases/download/release-3.8.0/s3ql-3.8.0.tar.gz.asc + validpgpkeys = ED31791B2C5C1613AF388B8AD113FCAC3C4E599F + sha256sums = bc5e05b6d929c0cde12b9966f426d7eae20d6f6137aa9291dec8873154b49631 + sha256sums = SKIP pkgname = s3ql - diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000000000..883ec344a43dc --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,1747 @@ +2021-11-07, S3QL 3.8.0 + + * The way to build the documentation has changed. Instead of running + `setup.py build_sphinx`, run `./build_docs.sh`. To generate PDF + documentation, follow this with `cd doc/pdf && make`. + + * The `s3ql_verify` tool is now able to detect the kind of corruption that + was introduced by the fsck.s3ql data corruption bug described below. + + * The new contrib/fixup_block_sizes.py tool is now available to fix most of + the issues caused by the fsck.s3ql bug described below. + +2021-06-03, S3QL 3.7.3 + + * Fixed a DATA CORRUPTION bug in fsck.s3ql that caused the recorded size of uploaded + dirty blocks to be rounded up to the next multiple of 512 bytes, effectively appending + up to 512 zero-bytes to the end of affected files. + + This problem was introduced in version 3.4.1 (released 2020-05-08) as part of a + seemingly very minor improvement to cache usage calculation. + + You can tell that a file has (likely) been affected from fsck.s3ql messages of the + form: + + WARNING: Writing dirty block of inode + + followed later by: + + WARNING: Size of inode () does not agree with number of blocks, \ + setting from to = 47. + + +2021-03-07, S3QL 3.7.1 + + * The tcp-timeout backend option of the B2 Backend works now. + + * mount.s3ql no longer crashes with "No Upload Threads available" when not running in + foreground. + + +2021-01-03, S3QL 3.7.0 + + * S3QL now requires Python 3.7 or newer. + + * S3QL now requires pyfuse3 of any version between 3.2.0 (inclusive) and 4.0 (exclusive). + + * S3QL now supports newer AWS S3 regions like eu-south-1. + + * mount.s3ql now again includes debugging information in its log output when + encountering an unexpected exception. This was broken in version 3.4.0, resulting in + mount.s3ql seemingly terminating at random in such a situation. + + * mount.s3ql now properly handles SIGTERM (instead of crashing). This means it exits as + quickly as possible without data corruption. For a proper unmount, always use + `umount.s3ql`, `umount`, or `fusermount -u` and wait for the mount.s3ql process to + terminate. + + * More robust expired auth token detection for Google Storage. + + +2020-11-09, S3QL 3.6.0 + + * Added ability to specify domain-name, project-domain-name and tenant-name as options + for the OpenStack Swift (Keystone v3) backend for providers that prefer name to id. + + * The open() syscall supports the O_TRUNC flag now. + + * `mount.s3ql` now exits gracefully on CTRL-C (INT signal) + + * `mount.s3ql` now supports the `--dirty-block-upload-delay` option to influence the + time before dirty blocks are written from the cache to the storage backend. + +2020-09-04, S3QL 3.5.1 + + * `s3qlctrl upload-meta` now works properly again (previously, only the first + invocation had an effect). + + * The O_TRUNC flag of the open() syscall is no longer silently ignored, but + the open() call fails with ENOTSUP. A proper implementation will hopefully + follow soon. + + +2020-07-15, S3QL 3.5.0 + + * S3QL now requires version 3.x rather than 1.x of the pyfuse3 module. + + * Triggering a metadata upload (either through ``s3qlctrl upload-meta`` or the + ``--metadata-upload-interval``) works again (since release 3.4.0, this would continue + to upload metadata continuously after the first trigger). + + * There is new backend for accessing Backblaze B2. + +2020-05-08, S3QL 3.4.1 + + * Fixed mount.s3ql "NoneType can't be used in 'await' expression" crash. + + +2020-03-19, S3QL 3.4.0 + + * There have been significant changes in the internal implementation. Asynchronous I/O + is now used in favor of threads in many places, and libfuse 3.x is used instead of + libfuse 2.x. + + * S3QL is now uses kernel-side writeback caching, which should significantly improve + write performance for small block sizes. + + * The dependency on the llfuse Python module has been dropped. A dependency on the `trio + `_ modules has been added instead. + + +2019-10-20, S3QL 3.3.2 + + * Fixed occasional crashes with "KeyError" when using Google Storage + backend. + + * Switched to semantic versioning. An increase in the major (first) version number will + indicate a backwards incompatible change (eg a change in the file system revision), a + change in the minor version (the second element) indicates new features, and a change + in the third element indicates bugfixes. + +2019-09-08, S3QL 3.3 + + * Added support for Keystone V3 authentication. + +2019-07-17, S3QL 3.2 + + * Added a `--compress` option to `fsck.s3ql`. + + * Minor improvements/fixes to `contrib/benchmark.py`. + + * Added support for INTELLIGENT_TIERING storage class to the S3 + backend. + +2019-03-31, S3QL 3.1 + + * Added a new `--fs-name` option to `mount.s3ql` allowing users + to specify a name for the fuse mount (This name is visible + in the first column of the system `mount` command output). + + * Added support for "Onezone-IA" to the S3 backend. + + * The `s3ql_oauth_client` command is working again now (it was + broken due to changes in Google's OAuth2 workflow). + + * Fixed refresh of Google Storage authentication tokens (expired + tokens were resulting in crashes). + +2019-02-09, S3QL 3.0 + + * Added a new `--systemd` option to simplify running mount.s3ql + as a systemd unit. + + * Dropped the `--upstart` option - upstart seems to be unused and + unmaintained. + + * Dropped support for legacy ("API key") authentication for Google + Storage. Only oauth2 is supported now. This was necessitated by + the switch to Google's native API (before S3QL was using Google's + S3 compatibility layer). + + * Command line options specified in the authinfo file (in particular + --backend-options) are now parsed correctly. + + * S3QL now uses python-cryptography instead of the (no longer + maintained) pycrypto module. + + * The Google Storage backend now supports Application Default + Credentials (ADC). To use this, install the google.auth module + and use ``adc`` as your backend login. + + * `umount.s3ql` now works correctly on systems where `ps` doesn't + accept a `-p` option (as long as /proc is available). + +2018-12-28, S3QL 2.33 + + * Fixed a sporadic test failure in t5_cache.py. + +2018-11-06, S3QL 2.32 + + * Fixed a potential bug in s3qlrm that would probably lead to a + filesystem crash (hasn't actually been observed though). + + * Clarified installation instructions: S3QL requires the official + systemd Python module, not the (third-party) module on PyPi. + + * Fixed occasional crashes with a "dugong.StateError" exception. + Thanks to Roger Gammans for extensive testing! + + * mount.s3ql and fsck.s3ql got a new `--keep-cache` option. If this + is specified, locally cached data will not be removed on unmount + (or after fsck) so that it can be re-used if the file system is + mounted again on the same system without having been mounted + elsewhere. + + * S3QL now considers all SSL-related errors as temporary and + automatically retries (instead of bailing out). + +2018-10-08, S3QL 2.31 + + * Object removal now makes more use of parallelism. + +2018-09-02, S3QL 2.30 + + * s3qladm has gained a new "recover-key" operation to restore the + file system passphrase from an offline copy. + + * Error messages in a few situations have been improved. + + * Re-generated C files with Cython 0.28.5 - this may fix some + compile problems under Python 3.7. + +2018-07-21, S3QL 2.29 + + * Removal of files that have not yet been uploaded to the backend is + now faster, with less data being uploaded only to be removed + shortly after. + + * Clarified NFS support in documentation. + + * contrib/clone_fs.py is now working again. + +2018-06-14, S3QL 2.28 + + * Fixed upgrade process for unencrypted filesystems (was broken in + 2.27 and 2.27.1). + + * mount.s3ql now checks /proc/mounts to determine if a storage url + is already mounted rather than just relying on flags in the + backend. + + * Fixed a busy loop, where mount.s3ql would perform normally but + consume much more CPU than needed. + + * Temporary SQLite files are now placed in --cachedir rather + than the system temp folder (typically /tmp). + +2018-06-09, S3QL 2.27.1 + + * Fixed a test failure in tests/t1_dump.py + +2018-06-09, S3QL 2.27 + + * The internal file system revision has changed. File systems + created with S3QL 2.27 or newer are not compatible with prior S3QL + versions. + + To update an existing file system to the newest revision, use the + 's3qladm upgrade' command. During the upgrade, the metadata for + all storage objects needs to be updated (so this may take a + while). + + * fsck.s3ql is now able to recover blocks that were in transit + during a file system crash. + + * Command line options that are common to all S3QL commands that + accept a storage URL (e.g. ``--cachedir`` or + ``--backend-options``) may now also been specified in the + *authinfo2* file by omitting the two leading dashes. + + * Fixed a data corruption issue when mount.s3ql is terminated + without being able to clean up (e.g. due to SIGKILL or the + computer being power cycled). + + * [SECURITY] Fixed a security vulnerability in the validation of object + metadata. All past versions of S3QL are affected by a replay + attack where a malicious person in control of the backend is able + to present old versions of the filesystem metadata database as + up-to-date. + + This could be used to e.g. make deleted files appear + again (though attempting to read them would result in an error), + or make newly created files appear to have vanished. + + Furthermore, an attacker could freely to set the "needs-fsck" + flag, the filesystem revison number, and filesystem block + size. Manipulating the filesystem revision number would result in + most S3QL commands refusing to work, attempting to upgrade the + revision with s3qladm would likely result in errors. Manipulating + the block size would result in mount.s3ql silently presenting file + incorrect file contents (despite the files being intact in the + backend): if the block size is increased, mount.s3ql will insert + regularly spaced gaps of zero-bytes. If the blocksize is + decreased, mount.s3ql will "cut out" regularly spaced windows of + data. + +2018-03-04, S3QL 2.26 + + * Added support for py.test 3.3.0 + * Improved metadata rotation performance. + +2018-01-05, S3QL 2.25 + + * s3qlctrl now accepts a new *dropcache* command. + * Fixed a race condition that resulted in mount.s3ql crashing with + "I/O operation on closed file". Thanks to www.imCode.com for + sponsoring this work! + +2017-11-04, S3QL 2.24 + + * fsck.s3ql now accepts a new --force-remote parameter. It does + dangerous things, and you should probably not use it. + + * When terminated by a signal (rather than by an unmount call), + mount.s3ql now terminates as quickly as possible (i.e., without + cleaning the cache), resulting in an unclean unmount. + + * The Swift backend feature detection is no longer affected by the + no-ssl backend option (which is intended only for connections to + the authentication server). + + * Accessing the cn-north-1 S3 storage region is now working. + +2017-08-18, S3QL 2.23 + + * Accessing the us-east-1 S3 storage region is now working again. + + * Retry for some additional SSL Errors. Unfortunately we're playing + whack-a-mole here. + + * The swift backend now dynamically detects the features supported + by the backend server. + + * The swift backend now supports optimized object copy and bulk + deletion (if supported by the server). + +2017-06-23, S3QL 2.22 + + * The storage URL for Amazon S3 buckets has changed. It now includes + the bucket's region. + + * Added support for AWS v4 authentication. Thanks to ibbchina.com + for sponsoring this work! + +2016-10-28, S3QL 2.21 + + * Added workaround for an issue where S3QL crashes with an + "ssl.SSLError: [SSL: BAD_WRITE_RETRY]" exception. This problem has + to be fixed in Python, but we now work around it by closing and + re-opening the connection when it occurs. + +2016-08-24, S3QL 2.20 + + * Fixed a problem with exporting S3QL file systems over NFS4, + where directory contents would appear in duplicate. + +2016-06-27, S3QL 2.19 + + * Fixed a crash when running s3qlstat on an almost empty + file system. + +2016-04-29, S3QL 2.18 + + * The `runtests.py` file has been dropped in favor of requiring + installation of py.test. + + * S3QL now uses semantic versioning. This means that + backwards-incompatible versions (i.e., versions that require an + upgrade of the file system revision) will be reflected in an + increase of the major version number, i.e. the next + backwards-incompatible version will have version 3.0. + + * Enabled several FUSE optimizations (most importantly, the + ``big_writes`` feature) that got accidentally disabled at + some point in the past. This should result in significant + performance improvements. + + Thanks to Simon Brown for the report! + + * Fixed a crash with "BUG ALERT: Dirty inode was destroyed". + +2016-03-08, S3QL 2.17.1 + + * Fixed a bug in the upgrade procedure that prevented file system + modifications when successive commands used the local metadata + cache. + + * Fixed a bug in the upgrade procedure that resulted in corrupted + file [acm]times when there was no cached metadata available for + both s3qladm and the first fsck.s3ql/mount.s3ql invocation after + the upgrade. + +2016-03-08, S3QL 2.17 + + * The internal file system revision has changed. File systems + created with S3QL 2.17 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest + revision, use the `s3qladm upgrade` command. + + * Due to a bug in the FUSE kernel module + (cf. https://github.com/libfuse/libfuse/issues/23) S3QL does not + (and did not) honor the write restriction bit. This limitation is + now documented. (It is unknown if the same problem exists when + running under BSD or OS-X). + + * S3QL now honors the setgid bit on directories (as opposed to just + storing them). + + * The `s3qlstat` command now accepts a --raw option to inhibit + pretty-printing of sizes. + + * The `s3qlctrl log` command is now working again. + + * The `s3qlctrl cachesize` command is now working again. + + * The s3 backend now supports the "infrequent access" storage class + (cf. https://aws.amazon.com/s3/storage-classes/) + + * The 's3qlstat' command now prints more information about cache + usage, including the number of cache entries and pending object + removals. + +2016-02-23, S3QL 2.16 + + * S3QL now supports (and requires) Python-LLFUSE 0.43 or newer + (but not never than 1.0). + + * S3QL now retries when receiving a "ServiceUnavailable" error from + an S3 (or compatible) backend. + + * When retrying network requests, the back-off time now includes a + random factor to avoid accidentally flooding the server with too + many simultanuous requests. + + * Fixed a problem with fsck.s3ql aborting with a "ConstraintError" + in some situations. + + * Fixed a problem with fsck.s3ql repeatedly renaming a file in + /lost+found because it may lack data. + + * In the authinfo2 file, storage urls for the local backend may now + be specified with a trailing slash. + +2015-09-18, S3QL 2.15 + + * Fixed a compatibility problem with recent versions of + Python-LLFUSE (0.41 and newer). + + * S3QL now supports using special characters in the prefix + for S3 buckets. Previously, characters that would need to + be percent-encoded when part of an URL (like + or %) would + result in a crash. + +2015-07-31, S3QL 2.14 + + * S3QL now longer crashes when the server sends a zero-length + response that claims to be XML. + + * The default tcp-timeout has been increased from 10 seconds to 20 + seconds. + + * Fixed test failures when FUSE is not available, tests should now + again be skipped. + + * S3QL now tracks the frequency of server errors and logs warnings + if they occur more often than 5 times per second over a period of + at least 60 seconds. Previously, retries would only be logged + if the same request has been retried more than 3 times. + + * The "s3qlctrl cachesize" command is now working again (previously + it would appear to succeed, but not actually change the cache + size). + + * 404 errors when retrying a delete request are now ignored - most + likely they mean that the server actually received and executed + the previous try. + + * Fixed an issue with fsck.s3ql crashing when trying to create a + file with a very long name in /lost+found. + +2015-01-31, S3QL 2.13 + + * The internal file system revision has changed. File systems + created with S3QL 2.13 or newer are not compatible with prior S3QL + versions. + + To update an existing file system to the newest revision, use the + 's3qladm upgrade' command. During the upgrade, all storage objects + that were created by S3QL 1.1 or earlier will be downloaded and + re-uploaded. For objects created by S3QL 1.1.2 or later, the + upgrade will only affect the storage object's metadata (so no + contents need to be transferred). + + * SECURITY ADVISORY: Previous mkfs.s3ql versions used /dev/urandom + to generate the master key when creating a new encrypted file + system. However, /dev/urandom is not guaranteed to provide the + 256-bits of entropy requested by S3QL. This may have allowed an + attacker to predict parts of the master key in situations where + little entropy is available (e.g. right after system boot, or if a + different program has previously drained the entropy pool). + + Note that the master key is not affected when the file system + passphrase is changed. The only way to recover from a potentially + compromised master key is to create a new file system and copy + over all data. + + * When creating new file systems, the master key is now generated by + reading /dev/random rather than /dev/urandom to ensure sufficient + entropy. + + * The 'no-ssl' option for the swift backend is now used only when + connecting to the authentication server. Whether SSL is used when + connecting to the storage server is determined by the + authentication server. + + * Fixed a crash when using the "disable-expect100" swift backend + option. + + * Fixed a race condition that could lead to a "KeyError" crash + when refreshing a Google Storage OAuth2 access token. + + * Fixed a race condition that could lead to a hanging mount.s3ql + process and hanging test_thread_hang() unit test. + + * Updated internal metadata storage and checksum format. The old + format was difficult to secure against malicious backend data and + could have resulted in false-positive checksum mismatches with + future or different Python interpreters. + + * Sizes (e.g. in the s3qlstat output) are now always reported + with three significant digits. + + * Fixed a bug that caused fsck.s3ql to either abort with a + "apsw.ConstraintError" or to incorrectly consider storage + objects as missing when the connection to remote server is + interrupted while retrieving the object list. + + * Storage urls without any prefix (e.g. s3://foo, but also + s3://foo/) are now using the same local cache + directory. Previously they would use different directories, + despite pointing to the same file system. + + (Note that this does not affect storage urls with a prefix, + s3://foo/bar and s3://foo/bar/ refer to different locations in the + bucket, and thus correctly use different cache directories). + + * Fixed a problem where mount.s3ql would crash when unmouting the + file system because it could not delete the cache directory. This + could happen when the file system was not unmounted cleanly, but + fsck.s3ql was then run on a different system (or using a different + cache directory). + + * S3QL now requires at least version 3.4 of the dugong Python + module. + +2014-11-09, S3QL 2.12 + + * The s3c backend has a new 'dumb-copy' option. If this option + is enabled, copy operations are assumed to have succeeded + as soon as the server returns a '200 OK' status, without + checking the contents of the response body. + + * The swift and s3c backends have a new 'disable-expect100' to + disable support for the 'Expect: continue' header. Using this + option allows S3QL to work with storage servers without proper + HTTP 1.1 support, but may decrease performance as object data will + be transmitted to the server more than once in some circumstances. + + * contrib/benchmark.py is now working again. + + * The `tcp-timeout` backend option is now actually working + instead of resulting in a "TypeError". + + * Fixed a problem where saving metadata would fail with + "ValueError: Can't dump NULL values". + + * s3qlstat now also gives information about cache usage. + +2014-09-04, S3QL 2.11.1 + + * By popular demand, mount.s3ql is now able to daemonize again + and also does so by default. + + * Un-encrypted file systems (created with mkfs.s3ql --plain) can now + be upgraded to the newest revision. + + * S3QL now automatically retries a request if a backend server + reports a 408 (Request Timeout) status code. + + * The Google Storage backend now copes better with server errors + when refreshing an OAuth2 access token. + +2014-08-27, S3QL 2.11 + + * SECURITY UPDATE (CVE-2014-0485). + + A remote code execution vulnerability was fixed. + + An attacker with control over the communication with the storage + backend or the ability to manipulate the data stored in the + backend was able to trigger execution of arbitrary code by + mount.s3ql, fsck.s3ql, mkfs.s3ql, s3qladm and s3ql_verify. Both + encrypted and unencrypted file systems were vulnerable. + + * s3ql_verify no longer crashes when checking an empty file system. + + * Fixed a crash when using Google OAuth2 and the first request after + the access token has expired is a write request. + + * mount.s3ql now supports systemd-style readyness notification. To + use this feature, make sure that you have the 'systemd' module + installed for Python 3. + + This feature has not been tested, if you use it or encounter + problems, please report back. + + * Fixed a race condition that could cause tests/t5_failsafe.py to + fail. + + * mount.s3ql no longer daemonizes on its own. With a modern init + system this should no longer be necessary, and when running + mount.s3ql from the command line the shell can be used to put the + process into background. + + * There is a new --backend-options parameter that can be used to + pass backend-specific options to any S3QL command that accepts a + storage url. + + * The --no-ssl and --ssl-ca-path parameters have been removed. For + those backends were these parameters make sense, you can use the + backend options of the same name instead (e.g. instead of + `--no-ssl` use `--backend-options no-ssl`). + + * Several backends now accept a `tcp-timeout` option. If S3QL is + unable to communicate with the remote server for longer than this + period, the TCP/IP connection is re-established. + + * The Amazon S3 backend now accepts a 'sse' option to enable server + side encryption. Both costs & benefits of S3 server side + encryption are probably very small, and this option does *not* + affect any client side encryption performed by S3QL itself. + + * The Amazon S3 backend now accepts a 'rrs' option to enable reduced + redundancy storage for any newly created objects. + +2014-07-28, S3QL 2.10.1 + + * 'setup.py install' now also installs the s3ql_verify command. + +2014-07-27, S3QL 2.10 + + * The internal file system revision has changed. File systems + created with S3QL 2.10 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest + revision, use the 's3qladm upgrade' command. + + It is strongly recommended to run the (new) s3ql_verify command + with the --data option at shortly after the upgrade. This is + necessary to ensure that the upgrade to the next (2.11) S3QL + release will run smoothly. + + * The User's Guide now contains a description of the possible + failure modes of mount.s3ql. + + * The --debug command line parameter now generates a bit less + output by default, and there is an additional --debug-module + parameter to activate additional messages. + + * When using encryption, S3QL now checks that a storage object's + key corresponds to the data stored under it. + + The lack of this check in previous versions allowed an attacker + with control over the storage server to interchange blocks within + the same file system (which would have resulted in apparent data + corruption in the file system). Targeted modification of specific + files, however, would have been very unlikely, because the + interchange of blocks had to be done blindly (with the attacker + not knowing which file any block belongs to, nor what its contents + are). + + Fixes https://bitbucket.org/nikratio/s3ql/issue/52/. + + * S3QL now aborts immediately instead of retrying if the server + storage server reports that the local clock is skewed. + + * There is a new 's3ql_verify' command. This program retrieves and + checks every storage object to ensure that all objects are + available and have not been tampered with. In contrast to + fsck.s3ql, s3ql_verify does not trust the object list provided by + the storage server and actually attempts to download the objects + one by one. + + * S3QL now requires version 3.2 or newer of the dugong module. + +2014-06-28, S3QL 2.9 + + * Fix crash when using swift backend and server uses an + authentication URL other than /v1.0. + + * Fixed two test failures when running unit tests as root. + + * Fixed problems when receiving an HTTP error without a well-formed + XML body from the remote server (this may happen e.g. due to + failure in a transparent proxy or load balancer). + + * S3QL now depends on the defusedxml Python module + (https://pypi.python.org/pypi/defusedxml/). + + * S3QL is no longer vulnerable to DOS attacks from malicious backend + servers. Previously, a malicious server could induce S3QL to + consume arbitrary amounts of memory by recursive XML entity + expansion. + + * S3QL now supports Google OAuth2 authentication. To use it, specify + 'oauth2' as backend login, and a valid OAuth2 refresh token as + backend password. To obtain the refresh token, the (new) + s3ql_oauth_client command may be used. + + * S3QL now requires version 3.1 or newer of the dugong module. + + * In some cases, transmission errors when reading storage objects + from a backend may have been misreported as corrupted backend + objects. This has been fixed. + + * S3QL no longer crashes when data corruption occurs in the first + few bytes of an LZMA compressed storage object. + + * S3QL now honors the "Retry-After" HTTP header when receiving an + XML error from a storage server. + + * Fixed a crash that could occur when the remote server (or some + intermediate proxy) sends a non-XML error response. + + * mount.s3ql and fsck.s3ql now use different exit codes to indicate + different failure conditions. + +2014-03-29, S3QL 2.8.1 + + * No changes in S3QL itself. + + * The S3QL 2.8 tarball accidentally included a copy of the Python + dugong module. This has been fixed in the 2.8.1 release. + +2014-03-13, S3QL 2.8 + + * Fixed various problems with using a proxy server. + + * Sending SIGUSR1 to mount.s3ql now generates a stacktrace + (debugging feature). + + * When passing --installed to the test runner, S3QL commands + are now loaded from $PATH instead of the packages bin/ + directory. + + * The runtest.py script now comes with the correct shebang + (i.e., it can now be called as "./runtests.py" instead of + "python3 runtests.py"). + + * S3QL now requires the python dugong module + (https://bitbucket.org/nikratio/python-dugong) + + * Fixed a file system hang when all upload threads encounter + unexpected backend problems. + +2013-12-16, S3QL 2.7 + + * Fixed another race condition that could lead to mount.s3ql + crashing with `ValueError: I/O operation on closed file`. + + * S3QL no longer generates warning messages for the first two times + that it has to resend a request to the storage backend. If there + is no success after the second try, messages are emitted as before. + + * S3QL now stores multiple copies of the master encryption key to + allow recovery if the backend looses the object holding the + primary copy. To take advantage of this functionality for existing + file systems, change the file system passphrase with s3qladm. + + * Fixed problem with automatic cache size detection (mount.s3ql was + treating bytes as kilobytes). Thanks to GVormayr@gmail.com for the + patch! + + * Fixed "AttributeError in LegacyDecryptDecompressFilter" crash when + reading objects written by old S3QL versions. + + * Fixed a problem with umount.s3ql giving a strange error when + the mountpoint is still in use. + +2013-12-03, S3QL 2.6 + + * Fixed a problem with mount.s3ql crashing with `KeyError` in + `collections/abc.py`. + + * Fixed a problem with mount.s3ql crashing with `ValueError: I/O + operation on closed file`. + + * Fixed a race condition that could lead to data corruption when + multiple processes trigger download of the same data block + simultaneously. + + * Fixed a problem with S3QL crashing with "SSLEOFError" when an SSL + connection is terminated unexpectedly. + + * Increased deduplication performance. Previously, block checksums + were accidentally computed without parallelism. + + * Fixed a problem with mount.s3ql (incorrectly) reporting corrupted + data for compressed blocks of some specific sizes. Many thanks to + Balázs for extensive debugging of this problem. + + * Specifying --debug when using syslog or file logging no longer + suppresses informational messages to stdout. + + * Fixed build under MacOS X. Thanks to john@nachtimwald.com for the + patch! + + * mount.s3ql now autodetects a sensible defaults for the + --max-cache-entries and --cachesize options. + +2013-10-19, S3QL 2.5 + + * Removed a race condition that resulted in a crash when a file was + deleted while the upload was in progress, and the upload was then + restarted because of a connection problem. + + * Specifying bzip2 as the compression algorithm for mount.s3ql + no longer gives an error. + + * Fixed another race condition that could lead to the (apparent) + vanishing of the file system metadata when the mount.s3ql is + killed at an unfortunate time. + + * S3QL programs no longer create an empty file in the default log + file location if a custom log file is specified with --log. + + * contrib/benchmark.py is now working again - thanks to + davidthomson@gmail.com. + + * If a backend returns corrupted data, mount.s3ql no longer crashes + but switches to read-only operation. + + * Fixed a problem with error messages from mount.s3ql being lost, + resulting in a file system shutdown for apparently no reason. + +2013-08-27, S3QL 2.4 + + * Attempts to upgrade a file system that is already at the newest + revision no longer render the local metadata copy unusable. + + * Google storage bucket names containing dots no longer cause + problems with SSL certificate validation. + + * Fixed crash when statvfs() is called on an almost empty file + system (which may happen automatically when using some desktop + environments, resulting in an immediate crash after mounting an + empty filesystem). + + * There is a new "swiftks" backend for OpenStack/Swift servers that + supports keystone authentication and allows specification of the + desired storage region. + + * Eliminated a race condition that could lead to mount.s3ql crashing + with a "KeyError in collections/abc.py" under some circumstances. + + * Fixed a crash that could occur when trying to clean up after + receiving a connection or server error when uploading an object. + +2013-07-18, S3QL 2.3 + + * Reading the file system passphrase from the authinfo2 file + is now working again. + + * The precedence of the authinfo2 sections has been fixed to agree + with the documentation: later entries override earlier entries + rather than the other way around. + + * Fixed a problem with mount.s3ql not recognizing the upgraded + revision when `s3qladm upgrade` was run with a different + --cachedir argument. + + * Fixed a crash of the s3 backend when multiple objects are deleted + and the s3 connection needs to be re-established. + +2013-07-12, S3QL 2.2 + + * Eliminated a race condition that resulted in an apparent loss of + remote metadata when the mount.s3ql process was interrupted after + renaming the remote metadata but before having completely uploaded + the local metadata. + + * Attempting to run mount.s3ql with --allow-user but without having + the necessary permissions no longer results in the file system + being marked as needing fsck. + + * The S3 backend now deletes multiple storage objects using one + request if possible, reducing latency, traffic and number of + requests to S3. + + * S3QL is now able to detect server errors and redirections before + all data has been sent to the server. This avoids pointless data + transfers and should fix a problem with S3QL operating very slow + when operating on recently created storage buckets/containers. + + * There is a new 'rackspace' backend that allows specification of + the object storage region in the storage URL. The rackspace + backend also uses a different authentication API than the swift + backend (which can still be used for Rackspace as well). When + using the 'rackspace' backend, use your regular Rackspace password + instead of the API key as the backend password. + + * The mount.s3ql --compress option now allows specification of the + compression level in addition to the compression algorithm. + +2013-06-14, S3QL 2.1 (beta) + + * S3QL now uses SSL connections by default. + + * S3QL now verifies server certificates when using SSL connections. + + * There is a new --no-ssl option that now works for every backend. + + * The Amazon S3 and Swift backends are now working again (regression + was introduced in 2.0-beta). + +2013-05-12, S3QL 2.0 (beta) + + * S3QL now requires Python 3.3 or newer. + + * S3QL now requires the pycrypto third-party Python module. + + * S3QL now requires at least version 0.39 of the llfuse Python + module. + + * Cycling of metadata backups when using the local backend now + takes much less time. + + * S3QL no longer requires the unittest2, pycryptopp and lzma + third-party Python modules. + + * It is no longer required that S3QL and Python APSW be linked to + the same SQLite shared library. + + * mount.s3ql: fixed crash when using --metadata-upload-interval 0. + + * Instead of completely unmounting the file system (and leaving the + mount point inaccessible) when backend data is missing or + corrupted, S3QL now returns an error and switches to read-only + operation. + +2013-04-09, S3QL 1.14 + + * Reduced CPU consumption when S3QL is idle. + + * S3QL now automatically retries requests when S3 reports an + "internal error" (this functionaly got accidentally broken + in S3QL 1.13). + + * Fixed a hang when using s3qlrm to delete a directory with + a very large number of sub-directories. + +2013-03-03, S3QL 1.13.2 + + * Allow S3 bucket names starting with a number. + + * Return EOPNOTSUP error on ACL operations. The previous value of + EINVAL works with FUSE, but confuses several applications. + +2013-02-24, S3QL 1.13.1 + + * Change error code returned to applications trying to set ACL + extended attributes. The previous error code of ENOSYS had the + unintended side effect that some FUSE versions gave up trying to + get/set any extended attributes at all. + +2013-02-23, S3QL 1.13 + + * Fixed issues when trying to access a directory at the root + of the file system that has recently been removed with s3qlrm. + (In that case the directory was no longer included in the + readdir() output, but it was still posssible to stat() it because + was still present in the kernel's dentry cache). + + * When trying to connect to the backend, S3QL now also retries at + increasing intervals if DNS appears to be unavailable (i.e., if + there's currently no network at all). This means that mount.s3ql + will no longer bail out when called without a network connection + but simply retry (printing warning messages in the process). + + * Work around undocumented "Timeout" reply from Google Storage and + retry instead of crashing. + + * Instead of failing with a bogus "Invalid credentials or skewed system + clock" error when trying to access a bucket with an invalid DNS + name, S3QL now prints a proper error message. + + (Such bucket names are still allowed when using the US Standard + regions, but generally not a good idea). + + * Removed ostensible ACL support. It was found that S3QL never kept + ACLs and file permission bits synchronized as mandated by POSIX. + This is due to a bug in the FUSE library and cannot easily be + fixed in S3QL. Consequently, S3QL no longer reports ACL support + unless a FUSE version with the necessary bugfix version is + installed. + + * Fixed a bug that caused malformed metadata to be written out when + using recent eglibc versions (the relevant change in eglibc + was introduced between eglibc versions 2.13 and 2.17). + + +2012-09-03, S3QL 1.12 + + * Be more verbose about how to remedy the situation when attempting + to upgrade file system needing fsck. + + * Fsck now detects and fixes directory entries which do not have an + entry type stored in their mode field. + +2012-05-04, S3QL 1.11.1 + + * Fixed crash when using S3 backend. + (Regression introduced in release 1.11). + + * Increase minimum reported file system size to 1 TB, and work + around df bug so that size is reported properly. + + * umount.s3ql now produces a more helpful error message if the + file system has crashed. + +2012-04-29, S3QL 1.11 + + * S3QL no longer uses the confusing "bucket" term (which is S3 + specific). The "bucket-passphrase" option in the authentication + file has been renamed to "fs-passphrase". + + * Metadata about stored objects is now split into multiple fields, + so that no field is longer than 256 bytes. This makes S3QL + compatible with a wider range of storage providers. + + Thanks to Stuart Wallace for reporting this + issue. + + * The SWIFT backend now retries if it receives 5xx error codes from + the remote server, and no longer fails when attempting to upload + data with an expired auth token. Thanks to Ken + for the report and initial patch! + +2012-02-24, S3QL 1.10 + + * The s3s, s3cs and gss backends have been removed. Use the new + --ssl option together with the s3, s3c and gs backends instead. + + * S3QL no longer keeps track of consistency guarantees for the + different backends. The increasing number of different storage + providers offering different regions, redundancy and availability + levels makes this no longer feasible. + + * The User's Guide contains a new section "Important Rules to Avoid + Losing Data". Reading it is strongly recommended. + +2012-01-21, S3QL 1.9 + + * Fixed a problem with file and directory time stamps + changing with the time zone of the mount.s3ql process. + + * Fixed a crash in contrib/benchmark.py and made output + (hopefully) easier to understand. + + * Fixed "Too many chained redirections" bug when trying to access + a new bucket immediately after creation. + + * When receiving 5xx errors from the server, S3QL now waits and + retries instead of aborting immediately. + + * contrib/s3ql_upstart.conf now supports running S3QL as an + ordinary user. + + * S3QL now has experimental, native support for OpenStack/SWIFT + servers. Thanks to the guys from www.bitbackup.de for providing + support and a test server! + + * S3QL now has experimental proxy support and will honor the + http_proxy and https_proxy environment variables. + +2011-12-06, S3QL 1.8.1 + + * Fixed direct updates from 1.6 to 1.8. + +2011-12-05, S3QL 1.8 + + * Fixed "ValueError: Can't dump NULL values" crash when file system + metadata was uploaded during heavy file system activity. + + * Deleting files no longer takes O(number of distinct filenames) + time (bug introduced in 1.7). + + * Improved s3qlcp and directory listing performance for very large + directories. + + * The --blocksize option of mkfs.s3ql has often been misunderstood, + leading people to create file systems with block sizes + significantly smaller than 1 MB. + + As a matter of fact, a small S3QL block size does *not* have any + advantage over a large block size when storing lots of small + files. A small block size, however, seriously degrades performance + when storing larger files. This is because S3QL is effectively + using a dynamic block size, and the --blocksize value merely + specifies an upper limit. + + To make this more clear, the --blocksize option has been renamed + to --max-obj-size. If you created your file system with a block + size of less than 1 MB, it is strongly recommended to recreate it + with a larger block size. Unfortunately, an in-place conversion of + the block size is not possible. + + * mount.s3ql has a new --min-obj-size option. In the future, files + smaller than the minimum object size may be combined into groups + that are stored as single objects in the storage backend. + + * Depend on python-llfuse 0.37. This fixes a problem with the file system + effectively blocking any other request while an s3qlrm, s3qllock + or s3qlcp instance is running. + + * Fixed some crashes happening under heavily concurrent file system + usage. + +2011-11-27, S3QL 1.7 + + * Extended attribute names are now de-duplicated. + + * Metadata is now stored in a custom, delta-encoded binary format and + then BZIP2 compressed, resulting in a 5-fold speedup when dumping + and compressing. + + * Inodes are now assigned sequentially rather than randomly, and old + inodes are not reused. This makes S3QL fully NFS compatible and + allows metadata to be stored much more efficiently, resulting in + a 4 to 8 fold decrease in compressed metadata size. + + * s3qlcp now also copies extended attributes. + + * s3qlcp no longer generates incorrect block reference counts when + copying a file that has identical blocks (i.e., that can be + de-duplicated within itself). + + * Eliminated a race condition in s3qlcp. When copying a file with + s3qlcp immediately after it was modified or created, it was + possible that s3qlcp would copy the new file attributes (size, + modification time, etc.) but the old data blocks. + + A copy of a newly created file would then contain only zero bytes, + while a copy of a modified file would look like the original but + contain data from before the modification. + + * "mkfs.s3ql --force" and "s3qladm clear" are working again. + +2011-11-20, S3QL 1.6 + + * fsck: gracefully recover if temporary indices already exist + (e.g. if previous fsck was interrupted). + + * Due not fail with "BadDigest" error when objects are only + partially retrieved from backend. + +2011-10-20, S3QL 1.5 + + * Fixed parsing of storage urls, s3s:// no longer generates bogus + error message. + + * Fixed support for prefix in storage urls. + + * Retry on timeout when transmitting data to remote server. + + * Do not free mount point when terminating due to unhandled + exception in background thread. + +2011-10-06, S3QL 1.4 + + * Metadata is now always LZMA compressed, mount.s3ql's --compress + option only affects compression of file contents. + + * Network errors that occur in the middle of a read operation + are now handled correctly as well. + + * s3qladm now uses cached metadata if available. + +2011-10-04, S3QL 1.3 + + * Fixed an s3qlcp crash happening when source or destination inode + number was a 64 bit value. + + * "Connection reset by peer" network errors are now handled + correctly. + + * fsck.s3ql can now renumber inodes. This is useful if the file + system was mounted using S3QL 1.2 or 1.1.x without the --nfs + option and can now no longer be accessed on 32bit systems. + + * Use only 32 bits for inodes, even when --nfs is not specified. + This ensures that file systems can be safely used by both 32 and + 64 bit systems. + +2011-09-28, S3QL 1.2 + + * Fixed a database problem that was responsible for file + system access becomer slower and slower the more data + was stored in the file system. + + * Fixed a race condition that could cause applications to get just + zero bytes when trying to read from a file that has just been + copied with s3qlcp. + +2011-09-20, S3QL 1.1.4 + + * Fixed a typo that caused errors when trying to remove any blocks + that have been committed to the backend. + + * Improved accuracy of s3qlstat during active file transfers + (but still not 100% accurate). + + * Fixed some theoretical deadlocks. + + * contrib/benchmark.py is now working again and also takes into + account the throughput from userspace to S3QL. + +2011-09-18, S3QL 1.1.3 + + * Fixed a race condition in the local backend that resulted in + errors of the form "[Errno 17] File exists: [bucket path]". + + * Added Google Storage backend. + + * Added backend for general, S3 compatible storage services. + + * Fixed a bug that caused S3QL to complain about the backend having + lost objects when trying to download an object before its upload + was completed. This could happen because locking was done based on + inode and block number rather than object id, and the + de-duplication feature can reuse an object for another inode + before the upload is completed. + + * Fixed a data corruption bug. If a data block was changed while + being uploaded to the backend, and a second, identical data block + was flushed while the upload was in progress, but before the first + block was changed, the second data block was linked to the + *modified* data. This happened because changes to an object in + transit were only checked for after the upload completed, leaving + a window in which the contents of the upload object did not agree + with its stored hash. + + This problem can be detected by verifying the hash values of all + stored data blocks. This procedure will automatically be done when + the file system is updated to the newest revision, but may take a + longer time since every object needs to be downloaded and checked. + +2011-09-08, S3QL 1.1.2 + + * The modules for communicating with the different storage providers + have been completely rewritten, resulting in improved performance, + more features and better maintainability. + + * S3 buckets can now be used with arbitrary prefixes, allowing to + store more than one S3QL file system in a bucket. + + * The format of the --authinfo file has changed. See the + documentation for details of the new format. To avoid breaking + backwards compatibility, the default file is now called authinfo2. + + * Network errors are now handled much more consistently. + + * There is a new --nfs option for mount.s3ql that needs to be used + when the S3QL file system will be exported over NFS. + + * The local backend now stores data and metadata in the same file, + so it needs only half as many files as before. + + * The --homedir option has been replaced by the more finely grained + --authfile, --cachedir and --log options. + + * S3QL can now log directly to syslog. + + * The sftp storage backend has been dropped. The recommended way to + use S3QL over ssh is to use sshfs + (http://fuse.sourceforge.net/sshfs.html) with S3QL's local + backend. + + * fsck now checks if all indices have been created. This avoids + a huge performance problem when mount.s3ql was interrupted + after downloading metadata, but before creating the indices. + +2011-07-23, S3QL 1.1 (development version) + + * Restructured metadata. This should also significantly reduce the + size of the SQLite database file. + + * Fixed license typo in file header comments, license + is GNU GPL Version 3, not LGPL. + + * Fixed problem with fsck.s3ql generating extraordinary long + filenames in /lost+found and then crashing. + + * When called as root, use umount rather than fusermount for + compatibility with FUSE4BSD. + +2011-05-20, S3QL 1.0.1 + + * Disabled WAL mode again for now because of unexpected problems + with s3qlcp, s3qllock and s3qlrm (performance going down orders + of magnitude, and *very* large *.db-wal file in ~/.s3ql). + +2011-05-13, S3QL 1.0 + + * S3QL has been declared stable after 2 years of beta-testing did + not reveal any data-critical bugs. + + * Fixed occasional assertion error when calling s3qlctrl flushcache + or unmounting the file system. + + * Fixed a race condition when a block is expired while it is in + transit but has already been modified again. + + * expire_backups.py no longer has an --init option, the state file + is created automatically if this operation is safe. Instead, there + is a --reconstruct-state option that can be used to try to + reconstruct a lost state file. + + * The size of symbolic links is now reported as the length of + the target instead of zero. This used to confuse revision + control systems like git and hg. + + * Added man pages for all S3QL commands. + +2011-02-04, S3QL 0.30 + + * S3QL now defaults to use unencrypted HTTP connections, which + significantly improves performance when using the S3 backend. For + an encrypted file system, all data is already encrypted anyway, + and authentication data is never transmitted in plain text even + for unencrypted file systems. Therefore, the use of SSL brings + little gain for most users. To force SSL usage, the new --ssl + option can be used. + + * mkfs.s3ql now has a --force option to overwrite an existing file + system. + +2010-12-30, S3QL 0.29 + + * The FUSE interface has been rewritten using Cython and factored + out into a separate package, http://code.google.com/p/python-llfuse/. + + This should result in easier installation, better performance and + better maintainability. + + +2010-12-19, S3QL 0.28 + + * "s3qlctrl upload-meta" now works even if the file system has been + mounted with --metadata-upload-interval=0. + + * File system metadata is now permanently cached locally. This + significantly reduces the time required to mount the file system. + + * The documentation is now also included in PDF format. + +2010-12-11, S3QL 0.27 + + * The authinfo file now supports passwords that include white space. + + * The s3qladm command can now be used to download metadata backups. + + * The --strip-meta option for mount.s3ql has been removed, redundant + data is now always stripped before upload. + + * mount.s3ql now has a --upstart option so that it can easily run as + an upstart job. An example job definition is included in + contrib/s3ql.conf. + + * s3qlctrl now has an 'upload-meta' command to trigger a metadata + upload while the file system is mounted. + + * Fixed a bug that preserved old data when truncating a file to + zero. If you ever had fsck errors of the form + + Size of inode [ddd] ([filename]) does not agree with number of + blocks + + then the affected files may contain additional, old data at the + end that is not supposed to be there. + + +2010-10-30, S3QL 0.26 + + * Fixed a problem with the sftp backend failing because it tries to + access a file locally. + + * Various minor bugfixes + +2010-09-28, S3QL 0.25 + + * A critical bug that could cause backups to be deleted too early + and potentially break the whole backup strategy has been fixed in + contrib/expire_backups.py. + + The new version has changed semantics that and also copes + significantly better when backups are made in irregular time + intervals. + + * S3QL should now respond with more consistent performance when + accessing the file system while s3qlcp, s3qllock or s3qlrm is + running at the same time. + + * When enabling debug output for the `UploadManager` module, S3QL + now logs detailed messages about de-duplication, upload and + compression performance. + +2010-09-18, S3QL 0.24 + + * Fixed a deadlock that could cause the mount.s3ql process to hang + around after umount. + + * Fixed a bug that caused S3QL to consider all downloaded blocks as + dirty and resulted in unneccessary checksum calculations on + expiration. + + * s3qlctrl can now change the log level at runtime. + + * s3qladm delete now also deletes any local stale cache files. + + * Periodic metadata upload can now be disabled completely by + specifying an interval of zero. + + +2010-09-03, S3QL 0.23 + + * Fixed problem with global_lock.py not being installed by setup.py + +2010-08-31, S3QL 0.22 + + * Fixed a bug that could cause file system errors when calling + s3qlrm on a lot of really large files. + + * The sftp backend is now significantly faster, thanks to skyang2009 + for the patch. + + * s3qlctrl can now change the cache size of a mounted file system. + +2010-08-09, S3QL 0.21 + + * The metadata is now only uploaded if there have been any changes. + + * mount.s3ql now supports several parallel compression and + encryption threads with the --compression-threads option. + + * S3QL now supports "immutable directories". This important new + feature can be used to protect backups from modification after + they have completed. See the User's Guide for details. + + * Using S3 RRS is now deprecated, see User's Guide for details. + + * fsck.s3ql now moves damaged files due to data lost by a backend + into /lost+found/ + + * expire_backups is no longer installed automatically and can now be + found in the contrib/ directory. + + * S3QL now comes with sample backup script in contrib/s3_backup.sh + + * Added contrib/pcp.py, an rsync wrapper to recursively copy + directory trees with several parallel rsync processes. + +2010-08-01, S3QL 0.20.1 + + * Hotfix for s3qladm upgrade. + +2010-08-01, S3QL 0.20 + + * Added contrib/make_dummy.py. This script creates a dummy copy of a + bucket that contains only the file system metadata. The resulting + dummy can be used for debugging. + + * Mounting with the local and sftp backends is now significantly + faster, because the object tree is no longer traversed completely. + + * Fixed a race condition that occasionally produced errors when + deleting large files (spanning multiple blocks). + + * The file system now stays responsive to other requests while + compressing blocks. + + * s3qlstat is now much faster since the size after de-duplication + does not need to be queried from the backend anymore but is stored + in the metadata. + + +2010-07-25, S3QL 0.19 + + * Fixed various smaller bugs, see Mercurial changelog for details. + +2010-07-11, S3QL 0.18 + + * Added --strip-meta option to mount.s3ql + + * Added --metadata-upload-interval option to mount.s3ql. This allows + to periodically upload updated metadata even while the file system + is mounted. + + * stat.s3ql, tune.s3ql and cp.s3ql have been renamed to s3qlstat, + s3qladm and s3qlcp respectively. + + * sftp backend is working again + + * Added the s3qlctrl command. + +2010-06-29, S3QL 0.17 + + * The local and sftp backends now spread their files into different + sub-directories. + + * Amazon S3 Reduced Redundancy Storage (RRS) is now supported. To + use it, use a storage url of the form s3rr:// instead of + s3://. + +2010-06-15, S3QL 0.16 + + * Fixed problem with readdir() returning entries twice or skipping + them if files are added or removed while readdir() is in progress. + + * Fixed build problem on Gentoo. + + * fsck.s3ql now does a rudimentary check if the file system is still + mounted to prevent checking a mounted file system. + + +2010-05-28, S3QL 0.15 + + * Fixed test cases to handle systems with low system clock + resolution. + + * Corrected installation instructions for Debian + + * mount.s3ql: instead of --bzip2, --zlib and --no-compress, there is + now just one option --compress=. + + * File system metadata is now uploaded with all indices. This makes + mounting the file system much faster. Only if LZMA compression has + been chosen, indices are stripped for storage and regenerated on + the next mount. + +2010-05-14, S3QL 0.14 + + * fsck now detects if a cached block is dirty and commits only dirty + blocks to the backend. + + * Installation in Debian and Ubuntu is now much simpler, it can be + done completely with aptitude. + +2010-05-04, S3QL 0.13 + + * S3QL now works with Ubuntu Karmic / 10.04 LTS + + * The test command no longer produces bogus error messages after all + tests have completed. + + * The umount.s3ql command now properly handles the 'fuser' output + with Kernel 2.6.32 (before it always refused to unmount, claiming + that the mount point was busy). + + * The compression can now be specified independently from the + encryption, so it is possible to have an unencrypted, but + compressed file system. + + * Apart from zlib, bzip2 and lzma, data can now also be stored + without any compression. + + * S3QL no longer emulates the . and .. directory entries since this + is not required by POSIX and makes the code much simpler. This + also means that the st_nlink value of a directory is not longer 2 + + number of subdirectories. + + * Fixed a bug that caused files to be listed with wrong sizes under + certain conditions. + + * Added `expire_backups` command and instructions for a simple + backup solution using S3QL and rsync. + +2010-04-27, S3QL 0.12 + + * fsck.s3ql now accepts a --batch option to not prompt for any user + input and requires a --force option to check the file system even + if it is marked as clean. + + * Fixed a bug in cp.s3ql that caused incorrect st_nlink values in + the copy. + + * fsck.s3ql has been even more optimized. + + * Fixed a problem with running out of file descriptors when lots of + objects are deleted. + + * Bucket encryption passwords can now also be stored in the + ~/.s3ql/authinfo file. + + * mount.s3ql doesn't complain any more if it receives any of the + standard mount(8) mount options. + + +2010-04-24, S3QL 0.11 + + * S3QL file system can now be mounted from /etc/fstab + + * Metadata now takes significantly less space. + + * Files with extended attributes can now be deleted. + + * Extended attributes can now be listed. + + * It is now possible to choose between zlib, BZip2 and LZMA + compression every time the file system is mounted. + + * Added contrib/benchmark.py to find out optimal compression method + for given network bandwidth. + + * fsck.s3ql no longer uses mknod(3) , since that lead to errors on + e.g. NFS mounted volumes under Fedora. + + * File access, modification and inode change times before 1972 are + now supported. + + * Fixed a deadlock when removing or overwriting files. + +2010-04-21, S3QL 0.10 + + * S3QL now depends on FUSE version 2.8 + + * Cached blocks are now flushed to the backend as soon as they have + not been accessed for more than 10 seconds. + + * The setup script now automatically checks and, if necessary, + installs the Python module dependencies. + + * mkfs.s3ql now creates compressed and encrypted file systems by + default. + + * Data is now compressed with LZMA instead of Bzip2. + + * Data compression and data upload is now done in parallel, so the + full network can bandwidth is continuously without breaks for data + compression. + + * fsck.s3ql is now several orders of magnitude faster. The code has + been optimized and some special database indices are now + precalculated. + + * When running cp.s3ql, the attributes of the target directory are + now immediately refreshed (so that e.g. ls immediately shows the + correct number of hard links). + + * File removal is now significantly faster, since the network + transactions are carried out asynchronously as part of the cache + expiration. + + * mount.s3ql no longer tries to create files with mknod(), since + that lead to errors on NFS mounted volumes under Fedora. + + * This releases includes a lot of new code, so it may be slightly + more beta-ish than usual. + + +2010-04-04, S3QL 0.9 + + * The --cachedir, --logfile, --awskey and --credfile options are + gone and have been replaced by --homedir. + + * ~/.awssecret is no longer read, instead there is a common file + with authentication data for all backends in ~/.s3ql/authinfo + + * The syntax for mounting S3 buckets is now s3://bucketname instead + of just the bucket name + + * There is now an SFTP backend. Thanks to Ron Knapp + for most of the code. + + +2010-03-07, S3QL 0.8 + + * S3QL now uses Python's default unittest.py instead of shipping its + own. + + * Most documentation has been moved from the Wiki into the tarball, + so that it always corresponds to the correct version. + + * setuptools is now used for installation. This allows .egg + creation, dependency resolution and generation of the HTML + documentation. + + * The S3 part of boto has been integrated into S3QL. + +2010-02-22, beta7 + + * mount.s3ql no longer chdirs into / when daemonizing but into the + cache directory. + + * Added example backup script in contrib/backup.py + + * tune.s3ql --change-passphrase is working again + + * Added testcase for tune.s3ql --change-passphrase + + * Internal S3 errors are now handled properly by retrying the + upload. + +2010-02-19, beta6 + + * tune.s3ql --copy is now *really* working properly (brrr) + +2010-02-19, beta5 + + * mkfs.s3ql now makes strict checks on the bucket name + + * Removed obsolete mount.s3ql_local from distribution + + * tune.s3ql --copy is now working properly + +2010-02-19, beta4 + + * tune.s3ql now has a --copy option to copy buckets + + * Storage location for new buckets can be specified in mkfs.s3ql + and tune.s3ql with --s3-location + + * Fixed a deadlock in mount.s3ql when using local buckets without + --fg + + * The local: bucket specifier is no longer artificially slow. + + * mount.s3ql: --allow_other is now working + +2010-02-04, beta3 + + * Fixed a deadlock that caused umount.s3ql to hang indefinitely + when mount was called without --fg + + * The '.' and '..' directory entries are no longer explicitly + stored in the database. + + * Rewrote the mechanism to handle delayed updates. Now it no + longer depends on a small object being propagated relatively + fast, but is able to handle arbitrary network outages. diff --git a/PKGBUILD b/PKGBUILD index e9704d9104cdc..78781d0b75865 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,43 +1,42 @@ -# Maintainer: Kazuki Sawada +# Maintainer: Luis Martinez +# Contributor: Kazuki Sawada # Contributor: Lalit Maganti # Contributor: Brendan MacDonell pkgname=s3ql -pkgver=3.7.1 +pkgver=3.8.0 pkgrel=1 pkgdesc="A full-featured file system for online data storage." -arch=('i686' 'x86_64') +arch=('x86_64') url="https://github.com/s3ql/s3ql/" license=('GPL3') depends=( - 'python' - 'python-cryptography' - 'python-defusedxml' - 'python-apsw' - 'python-trio' - 'python-pyfuse3' - 'python-dugong' - 'python-systemd' - 'python-requests' - 'python-google-auth' - 'python-google-auth-oauthlib' -) -makedepends=('gcc') -install="s3ql.install" -source=(https://github.com/s3ql/s3ql/releases/download/release-$pkgver/s3ql-$pkgver.tar.bz2) -md5sums=('ef7062f834dd8ee0614b78cb6382e9a0') + 'python-cryptography' + 'python-defusedxml' + 'python-apsw' + 'python-trio' + 'python-pyfuse3' + 'python-dugong' + 'python-systemd' + 'python-requests' + 'python-google-auth' + 'python-google-auth-oauthlib') +install=s3ql.install +changelog=CHANGELOG +source=( + "$pkgname-$pkgver.tar.gz::$url/releases/download/release-$pkgver/$pkgname-$pkgver.tar.gz" + "$pkgname-$pkgver.tar.gz.asc::$url/releases/download/release-$pkgver/$pkgname-$pkgver.tar.gz.asc") +sha256sums=('bc5e05b6d929c0cde12b9966f426d7eae20d6f6137aa9291dec8873154b49631' + 'SKIP') +validpgpkeys=('ED31791B2C5C1613AF388B8AD113FCAC3C4E599F') # Nikolaus Rath + +build() { + cd "$pkgname-$pkgver" + python setup.py build +} package() { - cd "$srcdir/$pkgname-$pkgver" - - python setup.py install --root="$pkgdir" --optimize=1 - - install -d -m 755 "$pkgdir/usr/share/doc/$pkgname" - install -m 744 doc/manual.pdf "$pkgdir/usr/share/doc/$pkgname/manual.pdf" - - # Install the text versions of the docs and contrib. These can be recursive, so - # `install' isn't particularly useful. - cp -r contrib rst "$pkgdir/usr/share/doc/$pkgname/" - chown -R root:root "$pkgdir/usr/share/doc/$pkgname/" - find "$pkgdir/usr/share/doc/$pkgname" -type f -exec chmod 'g-w,g+r,o-w,o+r' '{}' ';' + export PYTHONHASHSEED=0 + cd "$pkgname-$pkgver" + python setup.py install --root="$pkgdir" --optimize=1 --skip-build }