From b7fc0ff09d24b372dc04b0c02b80659c0a66fdfe Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 01:45:42 +0000 Subject: [PATCH 1/8] Squashfs: extend decompressor framework to handle compression options Extend decompressor framework to handle compression options stored in the filesystem. These options can be used by the relevant decompressor at initialisation time to over-ride defaults. The presence of compression options in the filesystem is indicated by the COMP_OPT filesystem flag. If present the data is read from the filesystem and passed to the decompressor init function. The decompressor init function signature has been extended to take this data. Also update the init function signature in the glib, lzo and xz decompressor wrappers. Signed-off-by: Phillip Lougher --- fs/squashfs/decompressor.c | 34 ++++++++++++++++++++++++++++++++++ fs/squashfs/decompressor.h | 7 +------ fs/squashfs/lzo_wrapper.c | 4 ++-- fs/squashfs/squashfs.h | 1 + fs/squashfs/squashfs_fs.h | 4 ++++ fs/squashfs/super.c | 11 +++++++---- fs/squashfs/xz_wrapper.c | 5 +++-- fs/squashfs/zlib_wrapper.c | 4 ++-- 8 files changed, 54 insertions(+), 16 deletions(-) diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c index a5940e54c4dd..e921bd213738 100644 --- a/fs/squashfs/decompressor.c +++ b/fs/squashfs/decompressor.c @@ -23,6 +23,7 @@ #include #include +#include #include #include "squashfs_fs.h" @@ -74,3 +75,36 @@ const struct squashfs_decompressor *squashfs_lookup_decompressor(int id) return decompressor[i]; } + + +void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + void *strm, *buffer = NULL; + int length = 0; + + /* + * Read decompressor specific options from file system if present + */ + if (SQUASHFS_COMP_OPTS(flags)) { + buffer = kmalloc(PAGE_CACHE_SIZE, GFP_KERNEL); + if (buffer == NULL) + return ERR_PTR(-ENOMEM); + + length = squashfs_read_data(sb, &buffer, + sizeof(struct squashfs_super_block), 0, NULL, + PAGE_CACHE_SIZE, 1); + + if (length < 0) { + strm = ERR_PTR(length); + goto finished; + } + } + + strm = msblk->decompressor->init(msblk, buffer, length); + +finished: + kfree(buffer); + + return strm; +} diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h index 3b305a70f7aa..099745ad5691 100644 --- a/fs/squashfs/decompressor.h +++ b/fs/squashfs/decompressor.h @@ -24,7 +24,7 @@ */ struct squashfs_decompressor { - void *(*init)(struct squashfs_sb_info *); + void *(*init)(struct squashfs_sb_info *, void *, int); void (*free)(void *); int (*decompress)(struct squashfs_sb_info *, void **, struct buffer_head **, int, int, int, int, int); @@ -33,11 +33,6 @@ struct squashfs_decompressor { int supported; }; -static inline void *squashfs_decompressor_init(struct squashfs_sb_info *msblk) -{ - return msblk->decompressor->init(msblk); -} - static inline void squashfs_decompressor_free(struct squashfs_sb_info *msblk, void *s) { diff --git a/fs/squashfs/lzo_wrapper.c b/fs/squashfs/lzo_wrapper.c index 7da759e34c52..00f4dfc5f088 100644 --- a/fs/squashfs/lzo_wrapper.c +++ b/fs/squashfs/lzo_wrapper.c @@ -37,7 +37,7 @@ struct squashfs_lzo { void *output; }; -static void *lzo_init(struct squashfs_sb_info *msblk) +static void *lzo_init(struct squashfs_sb_info *msblk, void *buff, int len) { int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); @@ -58,7 +58,7 @@ failed2: failed: ERROR("Failed to allocate lzo workspace\n"); kfree(stream); - return NULL; + return ERR_PTR(-ENOMEM); } diff --git a/fs/squashfs/squashfs.h b/fs/squashfs/squashfs.h index ba729d808876..1f2e608b8785 100644 --- a/fs/squashfs/squashfs.h +++ b/fs/squashfs/squashfs.h @@ -48,6 +48,7 @@ extern int squashfs_read_table(struct super_block *, void *, u64, int); /* decompressor.c */ extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int); +extern void *squashfs_decompressor_init(struct super_block *, unsigned short); /* export.c */ extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h index 39533feffd6d..4582c568ef4d 100644 --- a/fs/squashfs/squashfs_fs.h +++ b/fs/squashfs/squashfs_fs.h @@ -57,6 +57,7 @@ #define SQUASHFS_ALWAYS_FRAG 5 #define SQUASHFS_DUPLICATE 6 #define SQUASHFS_EXPORT 7 +#define SQUASHFS_COMP_OPT 10 #define SQUASHFS_BIT(flag, bit) ((flag >> bit) & 1) @@ -81,6 +82,9 @@ #define SQUASHFS_EXPORTABLE(flags) SQUASHFS_BIT(flags, \ SQUASHFS_EXPORT) +#define SQUASHFS_COMP_OPTS(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_COMP_OPT) + /* Max number of types and file types */ #define SQUASHFS_DIR_TYPE 1 #define SQUASHFS_REG_TYPE 2 diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index 20700b9f2b4c..95467db71a8e 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -199,10 +199,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) err = -ENOMEM; - msblk->stream = squashfs_decompressor_init(msblk); - if (msblk->stream == NULL) - goto failed_mount; - msblk->block_cache = squashfs_cache_init("metadata", SQUASHFS_CACHED_BLKS, SQUASHFS_METADATA_SIZE); if (msblk->block_cache == NULL) @@ -215,6 +211,13 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) goto failed_mount; } + msblk->stream = squashfs_decompressor_init(sb, flags); + if (IS_ERR(msblk->stream)) { + err = PTR_ERR(msblk->stream); + msblk->stream = NULL; + goto failed_mount; + } + /* Allocate and read id index table */ msblk->id_table = squashfs_read_id_index_table(sb, le64_to_cpu(sblk->id_table_start), le16_to_cpu(sblk->no_ids)); diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index c4eb40018256..397adea72eb9 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -38,7 +38,8 @@ struct squashfs_xz { struct xz_buf buf; }; -static void *squashfs_xz_init(struct squashfs_sb_info *msblk) +static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff, + int len) { int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); @@ -55,7 +56,7 @@ static void *squashfs_xz_init(struct squashfs_sb_info *msblk) failed: ERROR("Failed to allocate xz workspace\n"); kfree(stream); - return NULL; + return ERR_PTR(-ENOMEM); } diff --git a/fs/squashfs/zlib_wrapper.c b/fs/squashfs/zlib_wrapper.c index 4661ae2b1cec..195b0d035e9b 100644 --- a/fs/squashfs/zlib_wrapper.c +++ b/fs/squashfs/zlib_wrapper.c @@ -32,7 +32,7 @@ #include "squashfs.h" #include "decompressor.h" -static void *zlib_init(struct squashfs_sb_info *dummy) +static void *zlib_init(struct squashfs_sb_info *dummy, void *buff, int len) { z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL); if (stream == NULL) @@ -47,7 +47,7 @@ static void *zlib_init(struct squashfs_sb_info *dummy) failed: ERROR("Failed to allocate zlib workspace\n"); kfree(stream); - return NULL; + return ERR_PTR(-ENOMEM); } From ff750311d30acc9564ef577050794953eee59f01 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 15:31:46 +0000 Subject: [PATCH 2/8] Squashfs: add compression options support to xz decompressor Pass the dictionary size used to compress datablocks. Using a dictionary size less than the block size saves memory overhead, in many cases without adversely affecting compression ratio. Signed-off-by: Phillip Lougher --- fs/squashfs/xz_wrapper.c | 51 +++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index 397adea72eb9..06d0d11b482a 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -38,25 +39,57 @@ struct squashfs_xz { struct xz_buf buf; }; +struct comp_opts { + __le32 dictionary_size; + __le32 flags; +}; + static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff, int len) { - int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); + struct comp_opts *comp_opts = buff; + struct squashfs_xz *stream; + int dict_size = msblk->block_size; + int err, n; - struct squashfs_xz *stream = kmalloc(sizeof(*stream), GFP_KERNEL); - if (stream == NULL) - goto failed; + if (comp_opts) { + /* check compressor options are the expected length */ + if (len < sizeof(*comp_opts)) { + err = -EIO; + goto failed; + } - stream->state = xz_dec_init(XZ_PREALLOC, block_size); - if (stream->state == NULL) + dict_size = le32_to_cpu(comp_opts->dictionary_size); + + /* the dictionary size should be 2^n or 2^n+2^(n+1) */ + n = ffs(dict_size) - 1; + if (dict_size != (1 << n) && dict_size != (1 << n) + + (1 << (n + 1))) { + err = -EIO; + goto failed; + } + } + + dict_size = max_t(int, dict_size, SQUASHFS_METADATA_SIZE); + + stream = kmalloc(sizeof(*stream), GFP_KERNEL); + if (stream == NULL) { + err = -ENOMEM; goto failed; + } + + stream->state = xz_dec_init(XZ_PREALLOC, dict_size); + if (stream->state == NULL) { + kfree(stream); + err = -ENOMEM; + goto failed; + } return stream; failed: - ERROR("Failed to allocate xz workspace\n"); - kfree(stream); - return ERR_PTR(-ENOMEM); + ERROR("Failed to initialise xz decompressor\n"); + return ERR_PTR(err); } From 681ffe2e4316801529fdafe379743d1d41fe2086 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 16:21:34 +0000 Subject: [PATCH 3/8] Squashfs: Update Kconfig help text to include xz compression Signed-off-by: Phillip Lougher --- fs/squashfs/Kconfig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig index aa68a8a31518..efc309fa3035 100644 --- a/fs/squashfs/Kconfig +++ b/fs/squashfs/Kconfig @@ -5,12 +5,12 @@ config SQUASHFS help Saying Y here includes support for SquashFS 4.0 (a Compressed Read-Only File System). Squashfs is a highly compressed read-only - filesystem for Linux. It uses zlib/lzo compression to compress both - files, inodes and directories. Inodes in the system are very small - and all blocks are packed to minimise data overhead. Block sizes - greater than 4K are supported up to a maximum of 1 Mbytes (default - block size 128K). SquashFS 4.0 supports 64 bit filesystems and files - (larger than 4GB), full uid/gid information, hard links and + filesystem for Linux. It uses zlib, lzo or xz compression to + compress both files, inodes and directories. Inodes in the system + are very small and all blocks are packed to minimise data overhead. + Block sizes greater than 4K are supported up to a maximum of 1 Mbytes + (default block size 128K). SquashFS 4.0 supports 64 bit filesystems + and files (larger than 4GB), full uid/gid information, hard links and timestamps. Squashfs is intended for general read-only filesystem use, for From 4c1d204cde1470863e78aa0e6f55246c57f15038 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 16:32:39 +0000 Subject: [PATCH 4/8] Squashfs: Update documentation to include compression options Signed-off-by: Phillip Lougher --- Documentation/filesystems/squashfs.txt | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Documentation/filesystems/squashfs.txt b/Documentation/filesystems/squashfs.txt index 66699afd66ca..2d78f1911844 100644 --- a/Documentation/filesystems/squashfs.txt +++ b/Documentation/filesystems/squashfs.txt @@ -59,12 +59,15 @@ obtained from this site also. 3. SQUASHFS FILESYSTEM DESIGN ----------------------------- -A squashfs filesystem consists of a maximum of eight parts, packed together on a byte -alignment: +A squashfs filesystem consists of a maximum of nine parts, packed together on a +byte alignment: --------------- | superblock | |---------------| + | compression | + | options | + |---------------| | datablocks | | & fragments | |---------------| @@ -91,7 +94,14 @@ the source directory, and checked for duplicates. Once all file data has been written the completed inode, directory, fragment, export and uid/gid lookup tables are written. -3.1 Inodes +3.1 Compression options +----------------------- + +Compressors can optionally support compression specific options (e.g. +dictionary size). If non-default compression options have been used, then +these are stored here. + +3.2 Inodes ---------- Metadata (inodes and directories) are compressed in 8Kbyte blocks. Each @@ -114,7 +124,7 @@ directory inode are defined: inodes optimised for frequently occurring regular files and directories, and extended types where extra information has to be stored. -3.2 Directories +3.3 Directories --------------- Like inodes, directories are packed into compressed metadata blocks, stored @@ -144,7 +154,7 @@ decompressed to do a lookup irrespective of the length of the directory. This scheme has the advantage that it doesn't require extra memory overhead and doesn't require much extra storage on disk. -3.3 File data +3.4 File data ------------- Regular files consist of a sequence of contiguous compressed blocks, and/or a @@ -163,7 +173,7 @@ Larger files use multiple slots, with 1.75 TiB files using all 8 slots. The index cache is designed to be memory efficient, and by default uses 16 KiB. -3.4 Fragment lookup table +3.5 Fragment lookup table ------------------------- Regular files can contain a fragment index which is mapped to a fragment @@ -173,7 +183,7 @@ A second index table is used to locate these. This second index table for speed of access (and because it is small) is read at mount time and cached in memory. -3.5 Uid/gid lookup table +3.6 Uid/gid lookup table ------------------------ For space efficiency regular files store uid and gid indexes, which are @@ -182,7 +192,7 @@ stored compressed into metadata blocks. A second index table is used to locate these. This second index table for speed of access (and because it is small) is read at mount time and cached in memory. -3.6 Export table +3.7 Export table ---------------- To enable Squashfs filesystems to be exportable (via NFS etc.) filesystems @@ -196,7 +206,7 @@ This table is stored compressed into metadata blocks. A second index table is used to locate these. This second index table for speed of access (and because it is small) is read at mount time and cached in memory. -3.7 Xattr table +3.8 Xattr table --------------- The xattr table contains extended attributes for each inode. The xattrs From 3ad126641c05f93d2fa153bb8ff762fb4cdbb885 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 16:42:20 +0000 Subject: [PATCH 5/8] Squashfs: xz_wrapper doesn't need to include squashfs_fs_i.h anymore Signed-off-by: Phillip Lougher --- fs/squashfs/xz_wrapper.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index 06d0d11b482a..aa47a286d1f8 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -30,7 +30,6 @@ #include "squashfs_fs.h" #include "squashfs_fs_sb.h" -#include "squashfs_fs_i.h" #include "squashfs.h" #include "decompressor.h" From 003a3194d36dc22c29cacda4d0c6fede2753c9d0 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Mon, 28 Feb 2011 18:43:48 +0000 Subject: [PATCH 6/8] Squashfs: wrap squashfs_mount() definition Squashfs_get_sb() to squashfs_mount() conversion (commit 152a0836) results in line over 80 characters. Signed-off-by: Phillip Lougher --- fs/squashfs/super.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index 95467db71a8e..5c8184c061a4 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -373,8 +373,8 @@ static void squashfs_put_super(struct super_block *sb) } -static struct dentry *squashfs_mount(struct file_system_type *fs_type, int flags, - const char *dev_name, void *data) +static struct dentry *squashfs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) { return mount_bdev(fs_type, flags, dev_name, data, squashfs_fill_super); } From 44cff8a9ee8a974f9e931df910688e7fc1f0b0f9 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 15 Mar 2011 22:09:55 +0000 Subject: [PATCH 7/8] Squashfs: handle corruption of directory structure Handle the rare case where a directory metadata block is uncompressed and corrupted, leading to a kernel oops in directory scanning (memcpy). Normally corruption is detected at the decompression stage and dealt with then, however, this will not happen if: - metadata isn't compressed (users can optionally request no metadata compression), or - the compressed metadata block was larger than the original, in which case the uncompressed version was used, or - the data was corrupt after decompression This patch fixes this by adding some sanity checks against known maximum values. Signed-off-by: Phillip Lougher --- fs/squashfs/dir.c | 9 +++++++++ fs/squashfs/namei.c | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/fs/squashfs/dir.c b/fs/squashfs/dir.c index 0dc340aa2be9..3f79cd1d0c19 100644 --- a/fs/squashfs/dir.c +++ b/fs/squashfs/dir.c @@ -172,6 +172,11 @@ static int squashfs_readdir(struct file *file, void *dirent, filldir_t filldir) length += sizeof(dirh); dir_count = le32_to_cpu(dirh.count) + 1; + + /* dir_count should never be larger than 256 */ + if (dir_count > 256) + goto failed_read; + while (dir_count--) { /* * Read directory entry. @@ -183,6 +188,10 @@ static int squashfs_readdir(struct file *file, void *dirent, filldir_t filldir) size = le16_to_cpu(dire->size) + 1; + /* size should never be larger than SQUASHFS_NAME_LEN */ + if (size > SQUASHFS_NAME_LEN) + goto failed_read; + err = squashfs_read_metadata(inode->i_sb, dire->name, &block, &offset, size); if (err < 0) diff --git a/fs/squashfs/namei.c b/fs/squashfs/namei.c index 7a9464d08cf6..5d922a6701ab 100644 --- a/fs/squashfs/namei.c +++ b/fs/squashfs/namei.c @@ -176,6 +176,11 @@ static struct dentry *squashfs_lookup(struct inode *dir, struct dentry *dentry, length += sizeof(dirh); dir_count = le32_to_cpu(dirh.count) + 1; + + /* dir_count should never be larger than 256 */ + if (dir_count > 256) + goto data_error; + while (dir_count--) { /* * Read directory entry. @@ -187,6 +192,10 @@ static struct dentry *squashfs_lookup(struct inode *dir, struct dentry *dentry, size = le16_to_cpu(dire->size) + 1; + /* size should never be larger than SQUASHFS_NAME_LEN */ + if (size > SQUASHFS_NAME_LEN) + goto data_error; + err = squashfs_read_metadata(dir->i_sb, dire->name, &block, &offset, size); if (err < 0) @@ -228,6 +237,9 @@ exit_lookup: d_add(dentry, inode); return ERR_PTR(0); +data_error: + err = -EIO; + read_failure: ERROR("Unable to read directory block [%llx:%x]\n", squashfs_i(dir)->start + msblk->directory_table, From 117a91e0f25fd7698e20ac3dfa62086be3dc82a3 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Tue, 22 Mar 2011 23:01:26 +0000 Subject: [PATCH 8/8] Squashfs: Use vmalloc rather than kmalloc for zlib workspace Bugzilla bug 31422 reports occasional "page allocation failure. order:4" at Squashfs mount time. Fix this by making zlib workspace allocation use vmalloc rather than kmalloc. Reported-by: Mehmet Giritli Signed-off-by: Phillip Lougher --- fs/squashfs/zlib_wrapper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/squashfs/zlib_wrapper.c b/fs/squashfs/zlib_wrapper.c index 195b0d035e9b..517688b32ffa 100644 --- a/fs/squashfs/zlib_wrapper.c +++ b/fs/squashfs/zlib_wrapper.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "squashfs_fs.h" #include "squashfs_fs_sb.h" @@ -37,8 +38,7 @@ static void *zlib_init(struct squashfs_sb_info *dummy, void *buff, int len) z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL); if (stream == NULL) goto failed; - stream->workspace = kmalloc(zlib_inflate_workspacesize(), - GFP_KERNEL); + stream->workspace = vmalloc(zlib_inflate_workspacesize()); if (stream->workspace == NULL) goto failed; @@ -56,7 +56,7 @@ static void zlib_free(void *strm) z_stream *stream = strm; if (stream) - kfree(stream->workspace); + vfree(stream->workspace); kfree(stream); }