-3610542890c4b8329d83361ba48fa874d27c97a8
+4231dd5cf0f04dd61b0b8bae44a357da8331c0e2
libbcachefs/io.c \
libbcachefs/journal.c \
libbcachefs/keylist.c \
+ libbcachefs/lz4_compress.c \
+ libbcachefs/lz4_decompress.c \
libbcachefs/migrate.c \
libbcachefs/move.c \
libbcachefs/movinggc.c \
if (!bucket)
return 0;
- unfixable_fsck_err_on(bucket < ca->mi.first_bucket ||
- bucket >= ca->mi.nbuckets, c,
- "bad prio bucket %llu", bucket);
+ if (mustfix_fsck_err_on(bucket < ca->mi.first_bucket ||
+ bucket >= ca->mi.nbuckets, c,
+ "bad prio bucket %llu", bucket))
+ return 0;
for (b = 0; b < ca->mi.nbuckets; b++, d++) {
if (d == end) {
ca->prio_last_buckets[bucket_nr] = bucket;
bucket_nr++;
- ret = prio_io(ca, bucket, REQ_OP_READ);
- if (bch2_dev_fatal_io_err_on(ret, ca,
- "prior read from bucket %llu",
- bucket) ||
- bch2_meta_read_fault("prio"))
- return -EIO;
+ ret = prio_io(ca, bucket, REQ_OP_READ) ||
+ bch2_meta_read_fault("prio");
+
+ if (mustfix_fsck_err_on(ret, c,
+ "IO error reading bucket gens (%i)",
+ ret))
+ return 0;
got = le64_to_cpu(p->magic);
expect = pset_magic(c);
- unfixable_fsck_err_on(got != expect, c,
- "bad magic (got %llu expect %llu) while reading prios from bucket %llu",
- got, expect, bucket);
+ if (mustfix_fsck_err_on(got != expect, c,
+ "bad magic (got %llu expect %llu) while reading prios from bucket %llu",
+ got, expect, bucket))
+ return 0;
- unfixable_fsck_err_on(PSET_CSUM_TYPE(p) >= BCH_CSUM_NR, c,
- "prio bucket with unknown csum type %llu bucket %lluu",
- PSET_CSUM_TYPE(p), bucket);
+ if (mustfix_fsck_err_on(PSET_CSUM_TYPE(p) >= BCH_CSUM_NR, c,
+ "prio bucket with unknown csum type %llu bucket %lluu",
+ PSET_CSUM_TYPE(p), bucket))
+ return 0;
csum = bch2_checksum(c, PSET_CSUM_TYPE(p),
prio_nonce(p),
(void *) p + sizeof(p->csum),
bucket_bytes(ca) - sizeof(p->csum));
- unfixable_fsck_err_on(bch2_crc_cmp(csum, p->csum), c,
- "bad checksum reading prios from bucket %llu",
- bucket);
+ if (fsck_err_on(bch2_crc_cmp(csum, p->csum), c,
+ "bad checksum reading prios from bucket %llu",
+ bucket))
+ return 0;
bch2_encrypt(c, PSET_CSUM_TYPE(p),
prio_nonce(p),
ca->buckets[b].prio[READ] = le16_to_cpu(d->prio[READ]);
ca->buckets[b].prio[WRITE] = le16_to_cpu(d->prio[WRITE]);
- bucket_cmpxchg(&ca->buckets[b], new, new.gen = d->gen);
+ bucket_cmpxchg(&ca->buckets[b], new, ({
+ new.gen = d->gen;
+ new.gen_valid = 1;
+ }));
}
mutex_lock(&c->bucket_lock);
struct bucket *g = PTR_BUCKET(ca, ptr);
struct bucket_mark new;
+ if (!g->mark.gen_valid) {
+ bucket_cmpxchg(g, new, ({
+ new.gen = ptr->gen;
+ new.gen_valid = 1;
+ }));
+ ca->need_prio_write = true;
+ }
+
if (fsck_err_on(gen_cmp(ptr->gen, g->mark.gen) > 0, c,
"%s ptr gen in the future: %u > %u",
type == BKEY_TYPE_BTREE
? "btree" : "data",
ptr->gen, g->mark.gen)) {
- bucket_cmpxchg(g, new, new.gen = ptr->gen);
- set_bit(BCH_FS_FIXED_GENS, &c->flags);
+ bucket_cmpxchg(g, new, ({
+ new.gen = ptr->gen;
+ new.gen_valid = 1;
+ }));
ca->need_prio_write = true;
+ set_bit(BCH_FS_FIXED_GENS, &c->flags);
}
}
return false;
}
+static void bch2_btree_node_read_complete(struct btree_read_bio *rb,
+ struct btree *b)
+{
+ struct bch_dev *ca = rb->pick.ca;
+
+ bio_put(&rb->bio);
+ percpu_ref_put(&ca->io_ref);
+ clear_btree_node_read_in_flight(b);
+ wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
+}
+
void bch2_btree_node_read_done(struct bch_fs *c, struct btree *b,
struct bch_dev *ca,
const struct bch_extent_ptr *ptr)
btree_node_reset_sib_u64s(b);
out:
- clear_btree_node_read_in_flight(b);
- wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
mempool_free(iter, &c->fill_iter);
return;
err:
bch2_btree_node_read_done(rb->c, rb->bio.bi_private,
rb->pick.ca, &rb->pick.ptr);
-
- percpu_ref_put(&rb->pick.ca->io_ref);
- bio_put(&rb->bio);
+ bch2_btree_node_read_complete(rb, rb->bio.bi_private);
}
static void btree_node_read_endio(struct bio *bio)
PTR_BUCKET_NR(rb->pick.ca, &rb->pick.ptr)) ||
bch2_meta_read_fault("btree")) {
set_btree_node_read_error(b);
- percpu_ref_put(&rb->pick.ca->io_ref);
- bio_put(bio);
+ bch2_btree_node_read_complete(rb, rb->bio.bi_private);
return;
}
struct bio *bio;
trace_btree_read(c, b);
- set_btree_node_read_in_flight(b);
pick = bch2_btree_pick_ptr(c, b);
if (bch2_fs_fatal_err_on(!pick.ca, c,
bio->bi_iter.bi_size = btree_bytes(c);
bch2_bio_map(bio, b->data);
+ set_btree_node_read_in_flight(b);
+
if (sync) {
submit_bio_wait(bio);
bch2_btree_node_read_done(c, b, pick.ca, &pick.ptr);
bch2_time_stats_update(&c->btree_read_time, start_time);
out:
- bio_put(bio);
- percpu_ref_put(&pick.ca->io_ref);
+ bch2_btree_node_read_complete(rb, b);
} else {
bio->bi_end_io = btree_node_read_endio;
bio->bi_private = b;
struct {
u8 gen;
+ unsigned gen_valid:1;
unsigned journal_seq_valid:1;
/*
#include "io.h"
#include "super-io.h"
-#include <linux/lz4.h>
+#include "lz4.h"
#include <linux/zlib.h>
enum bounced {
switch (crc.compression_type) {
case BCH_COMPRESSION_LZ4:
- ret = LZ4_decompress_safe(src_data, dst_data,
- src_len, dst_len);
-
- if (ret != dst_len) {
+ ret = lz4_decompress(src_data, &src_len,
+ dst_data, dst_len);
+ if (ret) {
ret = -EIO;
goto err;
}
switch (compression_type) {
case BCH_COMPRESSION_LZ4: {
void *workspace;
- int srclen = src->bi_iter.bi_size;
- ret = 0;
+
+ *dst_len = dst->bi_iter.bi_size;
+ *src_len = src->bi_iter.bi_size;
workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
- while (srclen > block_bytes(c) &&
- (ret = LZ4_compress_destSize(src_data, dst_data,
- &srclen, dst->bi_iter.bi_size,
- workspace)) &&
- (srclen & (block_bytes(c) - 1))) {
- /* Round down to nearest block and try again: */
- srclen = round_down(srclen, block_bytes(c));
+ while (*src_len > block_bytes(c) &&
+ (ret = lz4_compress(src_data, *src_len,
+ dst_data, dst_len,
+ workspace))) {
+ /*
+ * On error, the compressed data was bigger than
+ * dst_len, and -ret is the amount of data we were able
+ * to compress - round down to nearest block and try
+ * again:
+ */
+ BUG_ON(ret > 0);
+ BUG_ON(-ret >= *src_len);
+
+ *src_len = round_down(-ret, block_bytes(c));
}
mempool_free(workspace, &c->lz4_workspace_pool);
- if (!ret)
+ if (ret)
goto err;
-
- *src_len = srclen;
- *dst_len = ret;
break;
}
case BCH_COMPRESSION_GZIP: {
--- /dev/null
+#ifndef __LZ4_H__
+#define __LZ4_H__
+/*
+ * LZ4 Kernel Interface
+ *
+ * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#define LZ4_MEM_COMPRESS (16384)
+#define LZ4HC_MEM_COMPRESS (262144 + (2 * sizeof(unsigned char *)))
+
+/*
+ * lz4_compressbound()
+ * Provides the maximum size that LZ4 may output in a "worst case" scenario
+ * (input data not compressible)
+ */
+static inline size_t lz4_compressbound(size_t isize)
+{
+ return isize + (isize / 255) + 16;
+}
+
+/*
+ * lz4_compress()
+ * src : source address of the original data
+ * src_len : size of the original data
+ * dst : output buffer address of the compressed data
+ * This requires 'dst' of size LZ4_COMPRESSBOUND.
+ * dst_len : is the output size, which is returned after compress done
+ * workmem : address of the working memory.
+ * This requires 'workmem' of size LZ4_MEM_COMPRESS.
+ * return : Success if return 0
+ * Error if return (< 0)
+ * note : Destination buffer and workmem must be already allocated with
+ * the defined size.
+ */
+int lz4_compress(const unsigned char *src, size_t src_len,
+ unsigned char *dst, size_t *dst_len, void *wrkmem);
+
+ /*
+ * lz4hc_compress()
+ * src : source address of the original data
+ * src_len : size of the original data
+ * dst : output buffer address of the compressed data
+ * This requires 'dst' of size LZ4_COMPRESSBOUND.
+ * dst_len : is the output size, which is returned after compress done
+ * workmem : address of the working memory.
+ * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
+ * return : Success if return 0
+ * Error if return (< 0)
+ * note : Destination buffer and workmem must be already allocated with
+ * the defined size.
+ */
+int lz4hc_compress(const unsigned char *src, size_t src_len,
+ unsigned char *dst, size_t *dst_len, void *wrkmem);
+
+/*
+ * lz4_decompress()
+ * src : source address of the compressed data
+ * src_len : is the input size, whcih is returned after decompress done
+ * dest : output buffer address of the decompressed data
+ * actual_dest_len: is the size of uncompressed data, supposing it's known
+ * return : Success if return 0
+ * Error if return (< 0)
+ * note : Destination buffer must be already allocated.
+ * slightly faster than lz4_decompress_unknownoutputsize()
+ */
+int lz4_decompress(const unsigned char *src, size_t *src_len,
+ unsigned char *dest, size_t actual_dest_len);
+
+/*
+ * lz4_decompress_unknownoutputsize()
+ * src : source address of the compressed data
+ * src_len : is the input size, therefore the compressed size
+ * dest : output buffer address of the decompressed data
+ * dest_len: is the max size of the destination buffer, which is
+ * returned with actual size of decompressed data after
+ * decompress done
+ * return : Success if return 0
+ * Error if return (< 0)
+ * note : Destination buffer must be already allocated.
+ */
+int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
+ unsigned char *dest, size_t *dest_len);
+#endif
--- /dev/null
+/*
+ * LZ4 - Fast LZ compression algorithm
+ * Copyright (C) 2011-2012, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ * - LZ4 source repository : http://code.google.com/p/lz4/
+ *
+ * Changed for kernel use by:
+ * Chanho Min <chanho.min@lge.com>
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <asm/unaligned.h>
+#include "lz4.h"
+#include "lz4defs.h"
+
+#define LZ4_HASH_VALUE(p, _table) \
+ __HASH_VALUE(p, MEMORY_USAGE - ilog2(sizeof(_table[0])))
+
+struct lz4_hash_table {
+ const u8 *(*add)(const struct lz4_hash_table, const u8 *);
+ void *ctx;
+ const u8 *base;
+};
+
+#if __SIZEOF_POINTER__ == 4
+static inline const u8 *hash_table_add32(const struct lz4_hash_table hash,
+ const u8 *ip)
+{
+ const u8 **table = hash.ctx;
+
+ swap(table[LZ4_HASH_VALUE(ip, table)], ip);
+ return ip;
+}
+#else
+static inline const u8 *hash_table_add32(const struct lz4_hash_table hash,
+ const u8 *ip)
+{
+ u32 *table = hash.ctx;
+ size_t offset = ip - hash.base;
+
+ swap(table[LZ4_HASH_VALUE(ip, table)], offset);
+ return hash.base + offset;
+}
+#endif
+
+static inline const u8 *hash_table_add16(const struct lz4_hash_table hash,
+ const u8 *ip)
+{
+ u16 *table = hash.ctx;
+ size_t offset = ip - hash.base;
+
+ swap(table[LZ4_HASH_VALUE(ip, table)], offset);
+ return hash.base + offset;
+}
+
+static inline const u8 *find_match(const struct lz4_hash_table hash,
+ const u8 **ip, const u8 *anchor,
+ const u8 *start, const u8 *mflimit)
+{
+ int findmatchattempts = (1U << SKIPSTRENGTH) + 3;
+
+ while (*ip <= mflimit) {
+ const u8 *ref = hash.add(hash, *ip);
+
+ if (ref >= *ip - MAX_DISTANCE && A32(ref) == A32(*ip)) {
+ /* found match: */
+ while (*ip > anchor &&
+ ref > start &&
+ unlikely((*ip)[-1] == ref[-1])) {
+ (*ip)--;
+ ref--;
+ }
+
+ return ref;
+ }
+
+ *ip += findmatchattempts++ >> SKIPSTRENGTH;
+ }
+
+ return NULL;
+}
+
+static inline int length_len(unsigned length)
+{
+ return length / 255 + 1;
+}
+
+/*
+ * LZ4_compressCtx :
+ * -----------------
+ * Compress 'isize' bytes from 'source' into an output buffer 'dest' of
+ * maximum size 'maxOutputSize'. * If it cannot achieve it, compression
+ * will stop, and result of the function will be zero.
+ * return : the number of bytes written in buffer 'dest', or 0 if the
+ * compression fails
+ */
+static inline int lz4_compressctx(const struct lz4_hash_table hash,
+ const u8 *src, size_t src_len,
+ u8 *dst, size_t *dst_len)
+{
+ const u8 *ip = src, *anchor = ip, *ref;
+ const u8 *const iend = ip + src_len;
+ const u8 *const mflimit = iend - MFLIMIT;
+ const u8 *const matchlimit = iend - LASTLITERALS;
+ u8 *op = dst, *token;
+ u8 *const oend = op + *dst_len;
+ size_t literal_len, match_len, match_offset;
+
+ /* Init */
+ memset(hash.ctx, 0, LZ4_MEM_COMPRESS);
+ hash.add(hash, ip);
+
+ /* Always start with a literal: */
+ ip++;
+
+ while ((ref = find_match(hash, &ip, anchor, src, mflimit))) {
+ /*
+ * We found a match; @ip now points to the match and @ref points
+ * to the prior part of the input we matched with. Everything up
+ * to @anchor has been encoded; the range from @anchor to @ip
+ * didn't match and now has to be encoded as a literal:
+ */
+ literal_len = ip - anchor;
+ match_offset = ip - ref;
+
+ /* MINMATCH bytes already matched from find_match(): */
+ ip += MINMATCH;
+ ref += MINMATCH;
+ match_len = common_length(ip, ref, matchlimit);
+ ip += match_len;
+
+ /* check output limit */
+ if (unlikely(op +
+ 1 + /* token */
+ 2 + /* match ofset */
+ literal_len +
+ length_len(literal_len) +
+ length_len(match_len) +
+ LASTLITERALS > oend))
+ break;
+
+ token = op++;
+ *token = encode_length(&op, literal_len) << ML_BITS;
+ MEMCPY_ADVANCE_CHUNKED(op, anchor, literal_len);
+ PUT_LE16_ADVANCE(op, match_offset);
+ *token += encode_length(&op, match_len);
+
+ anchor = ip;
+ }
+
+ /* Encode remaining input as literal: */
+ literal_len = iend - anchor;
+ if (unlikely(op +
+ 1 +
+ literal_len +
+ length_len(literal_len) > oend)) {
+ /* Return how much would be able to fit: */
+ ssize_t remaining = oend - op;
+ ssize_t encoded = anchor - src;
+
+ remaining -= length_len(remaining) + 1;
+
+ return -max(encoded + remaining, 1L);
+ }
+
+ token = op++;
+ *token = encode_length(&op, literal_len) << ML_BITS;
+ MEMCPY_ADVANCE(op, anchor, literal_len);
+
+ /* End */
+ BUG_ON(op > oend);
+ *dst_len = op - dst;
+ return 0;
+}
+
+__attribute__((flatten))
+int lz4_compress(const unsigned char *src, size_t src_len,
+ unsigned char *dst, size_t *dst_len, void *wrkmem)
+{
+ if (src_len < LZ4_64KLIMIT) {
+ const struct lz4_hash_table hash = {
+ .add = hash_table_add16,
+ .ctx = wrkmem,
+ .base = src,
+ };
+
+ return lz4_compressctx(hash, src, src_len, dst, dst_len);
+ } else {
+ const struct lz4_hash_table hash = {
+ .add = hash_table_add32,
+ .ctx = wrkmem,
+ .base = src,
+ };
+
+ return lz4_compressctx(hash, src, src_len, dst, dst_len);
+ }
+}
+EXPORT_SYMBOL(lz4_compress);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("LZ4 compressor");
--- /dev/null
+/*
+ * LZ4 Decompressor for Linux kernel
+ *
+ * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
+ *
+ * Based on LZ4 implementation by Yann Collet.
+ *
+ * LZ4 - Fast LZ compression algorithm
+ * Copyright (C) 2011-2012, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ * - LZ4 source repository : http://code.google.com/p/lz4/
+ */
+
+#ifndef STATIC
+#include <linux/module.h>
+#include <linux/kernel.h>
+#endif
+
+#include "lz4.h"
+#include "lz4defs.h"
+
+static const int dec32table[8] = {0, 3, 2, 3, 0, 0, 0, 0};
+#if LZ4_ARCH64
+static const int dec64table[8] = {0, 0, 0, -1, 0, 1, 2, 3};
+#else
+static const int dec64table[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+#endif
+
+static inline size_t get_length(const u8 **ip, size_t length)
+{
+ if (length == LENGTH_LONG) {
+ size_t len;
+
+ do {
+ length += (len = *(*ip)++);
+ } while (len == 255);
+ }
+
+ return length;
+}
+
+static int lz4_uncompress(const u8 *source, u8 *dest, int osize)
+{
+ const u8 *ip = source;
+ const u8 *ref;
+ u8 *op = dest;
+ u8 * const oend = op + osize;
+ u8 *cpy;
+ unsigned token, offset;
+ ssize_t length;
+
+ while (1) {
+ /* get runlength */
+ token = *ip++;
+ length = get_length(&ip, token >> ML_BITS);
+
+ /* copy literals */
+ if (unlikely(op + length > oend - COPYLENGTH)) {
+ /*
+ * Error: not enough place for another match
+ * (min 4) + 5 literals
+ */
+ if (op + length != oend)
+ goto _output_error;
+
+ MEMCPY_ADVANCE(op, ip, length);
+ break; /* EOF */
+ }
+ MEMCPY_ADVANCE_CHUNKED(op, ip, length);
+
+ /* get match offset */
+ offset = GET_LE16_ADVANCE(ip);
+ ref = op - offset;
+
+ /* Error: offset create reference outside destination buffer */
+ if (unlikely(ref < (u8 *const) dest))
+ goto _output_error;
+
+ /* get match length */
+ length = get_length(&ip, token & ML_MASK);
+ length += MINMATCH;
+
+ /* copy first STEPSIZE bytes of match: */
+ if (unlikely(offset < STEPSIZE)) {
+ MEMCPY_ADVANCE_BYTES(op, ref, 4);
+ ref -= dec32table[offset];
+
+ memcpy(op, ref, 4);
+ op += STEPSIZE - 4;
+ ref -= dec64table[offset];
+ } else {
+ MEMCPY_ADVANCE(op, ref, STEPSIZE);
+ }
+ length -= STEPSIZE;
+ /*
+ * Note - length could have been < STEPSIZE; that's ok, length
+ * will now be negative and we'll just end up rewinding op:
+ */
+
+ /* copy rest of match: */
+ cpy = op + length;
+ if (cpy > oend - COPYLENGTH) {
+ /* Error: request to write beyond destination buffer */
+ if (cpy > oend ||
+ ref + COPYLENGTH > oend)
+ goto _output_error;
+#if !LZ4_ARCH64
+ if (op + COPYLENGTH > oend)
+ goto _output_error;
+#endif
+ MEMCPY_ADVANCE_CHUNKED_NOFIXUP(op, ref, oend - COPYLENGTH);
+ /* op could be > cpy here */
+ while (op < cpy)
+ *op++ = *ref++;
+ op = cpy;
+ /*
+ * Check EOF (should never happen, since last 5 bytes
+ * are supposed to be literals)
+ */
+ if (op == oend)
+ goto _output_error;
+ } else {
+ MEMCPY_ADVANCE_CHUNKED(op, ref, length);
+ }
+ }
+ /* end of decoding */
+ return ip - source;
+
+ /* write overflow error detected */
+_output_error:
+ return -1;
+}
+
+static inline ssize_t get_length_safe(const u8 **ip, ssize_t length)
+{
+ if (length == 15) {
+ size_t len;
+
+ do {
+ length += (len = *(*ip)++);
+ if (unlikely((ssize_t) length < 0))
+ return -1;
+
+ length += len;
+ } while (len == 255);
+ }
+
+ return length;
+}
+
+static int lz4_uncompress_unknownoutputsize(const u8 *source, u8 *dest,
+ int isize, size_t maxoutputsize)
+{
+ const u8 *ip = source;
+ const u8 *const iend = ip + isize;
+ const u8 *ref;
+ u8 *op = dest;
+ u8 * const oend = op + maxoutputsize;
+ u8 *cpy;
+ unsigned token, offset;
+ size_t length;
+
+ /* Main Loop */
+ while (ip < iend) {
+ /* get runlength */
+ token = *ip++;
+ length = get_length_safe(&ip, token >> ML_BITS);
+ if (unlikely((ssize_t) length < 0))
+ goto _output_error;
+
+ /* copy literals */
+ if ((op + length > oend - COPYLENGTH) ||
+ (ip + length > iend - COPYLENGTH)) {
+
+ if (op + length > oend)
+ goto _output_error;/* writes beyond buffer */
+
+ if (ip + length != iend)
+ goto _output_error;/*
+ * Error: LZ4 format requires
+ * to consume all input
+ * at this stage
+ */
+ MEMCPY_ADVANCE(op, ip, length);
+ break;/* Necessarily EOF, due to parsing restrictions */
+ }
+ MEMCPY_ADVANCE_CHUNKED(op, ip, length);
+
+ /* get match offset */
+ offset = GET_LE16_ADVANCE(ip);
+ ref = op - offset;
+
+ /* Error: offset create reference outside destination buffer */
+ if (ref < (u8 * const) dest)
+ goto _output_error;
+
+ /* get match length */
+ length = get_length_safe(&ip, token & ML_MASK);
+ if (unlikely((ssize_t) length < 0))
+ goto _output_error;
+
+ length += MINMATCH;
+
+ /* copy first STEPSIZE bytes of match: */
+ if (unlikely(offset < STEPSIZE)) {
+ MEMCPY_ADVANCE_BYTES(op, ref, 4);
+ ref -= dec32table[offset];
+
+ memcpy(op, ref, 4);
+ op += STEPSIZE - 4;
+ ref -= dec64table[offset];
+ } else {
+ MEMCPY_ADVANCE(op, ref, STEPSIZE);
+ }
+ length -= STEPSIZE;
+
+ /* copy rest of match: */
+ cpy = op + length;
+ if (cpy > oend - COPYLENGTH) {
+ /* Error: request to write beyond destination buffer */
+ if (cpy > oend ||
+ ref + COPYLENGTH > oend)
+ goto _output_error;
+#if !LZ4_ARCH64
+ if (op + COPYLENGTH > oend)
+ goto _output_error;
+#endif
+ MEMCPY_ADVANCE_CHUNKED_NOFIXUP(op, ref, oend - COPYLENGTH);
+ while (op < cpy)
+ *op++ = *ref++;
+ op = cpy;
+ /*
+ * Check EOF (should never happen, since last 5 bytes
+ * are supposed to be literals)
+ */
+ if (op == oend)
+ goto _output_error;
+ } else {
+ MEMCPY_ADVANCE_CHUNKED(op, ref, length);
+ }
+ }
+ /* end of decoding */
+ return op - dest;
+
+ /* write overflow error detected */
+_output_error:
+ return -1;
+}
+
+int lz4_decompress(const unsigned char *src, size_t *src_len,
+ unsigned char *dest, size_t actual_dest_len)
+{
+ int ret = -1;
+ int input_len = 0;
+
+ input_len = lz4_uncompress(src, dest, actual_dest_len);
+ if (input_len < 0)
+ goto exit_0;
+ *src_len = input_len;
+
+ return 0;
+exit_0:
+ return ret;
+}
+#ifndef STATIC
+EXPORT_SYMBOL(lz4_decompress);
+#endif
+
+int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
+ unsigned char *dest, size_t *dest_len)
+{
+ int ret = -1;
+ int out_len = 0;
+
+ out_len = lz4_uncompress_unknownoutputsize(src, dest, src_len,
+ *dest_len);
+ if (out_len < 0)
+ goto exit_0;
+ *dest_len = out_len;
+
+ return 0;
+exit_0:
+ return ret;
+}
+#ifndef STATIC
+EXPORT_SYMBOL(lz4_decompress_unknownoutputsize);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("LZ4 Decompressor");
+#endif
--- /dev/null
+/*
+ * lz4defs.h -- architecture specific defines
+ *
+ * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * Detects 64 bits mode
+ */
+#if defined(CONFIG_64BIT)
+#define LZ4_ARCH64 1
+#else
+#define LZ4_ARCH64 0
+#endif
+
+#include <asm/unaligned.h>
+#include <linux/log2.h>
+#include <linux/string.h>
+
+#define A32(_p) get_unaligned((u32 *) (_p))
+#define A16(_p) get_unaligned((u16 *) (_p))
+
+#define GET_LE16_ADVANCE(_src) \
+({ \
+ u16 _r = get_unaligned_le16(_src); \
+ (_src) += 2; \
+ _r; \
+})
+
+#define PUT_LE16_ADVANCE(_dst, _v) \
+do { \
+ put_unaligned_le16((_v), (_dst)); \
+ (_dst) += 2; \
+} while (0)
+
+#define LENGTH_LONG 15
+#define COPYLENGTH 8
+#define ML_BITS 4
+#define ML_MASK ((1U << ML_BITS) - 1)
+#define RUN_BITS (8 - ML_BITS)
+#define RUN_MASK ((1U << RUN_BITS) - 1)
+#define MEMORY_USAGE 14
+#define MINMATCH 4
+#define SKIPSTRENGTH 6
+#define LASTLITERALS 5
+#define MFLIMIT (COPYLENGTH + MINMATCH)
+#define MINLENGTH (MFLIMIT + 1)
+#define MAXD_LOG 16
+#define MAXD (1 << MAXD_LOG)
+#define MAXD_MASK (u32)(MAXD - 1)
+#define MAX_DISTANCE (MAXD - 1)
+#define HASH_LOG (MAXD_LOG - 1)
+#define HASHTABLESIZE (1 << HASH_LOG)
+#define MAX_NB_ATTEMPTS 256
+#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
+#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT - 1))
+
+#define __HASH_VALUE(p, bits) \
+ (((A32(p)) * 2654435761U) >> (32 - (bits)))
+
+#define HASH_VALUE(p) __HASH_VALUE(p, HASH_LOG)
+
+#define MEMCPY_ADVANCE(_dst, _src, length) \
+do { \
+ typeof(length) _length = (length); \
+ memcpy(_dst, _src, _length); \
+ _src += _length; \
+ _dst += _length; \
+} while (0)
+
+#define MEMCPY_ADVANCE_BYTES(_dst, _src, _length) \
+do { \
+ const u8 *_end = (_src) + (_length); \
+ while ((_src) < _end) \
+ *_dst++ = *_src++; \
+} while (0)
+
+#define STEPSIZE __SIZEOF_LONG__
+
+#define LZ4_COPYPACKET(_src, _dst) \
+do { \
+ MEMCPY_ADVANCE(_dst, _src, STEPSIZE); \
+ MEMCPY_ADVANCE(_dst, _src, COPYLENGTH - STEPSIZE);\
+} while (0)
+
+/*
+ * Equivalent to MEMCPY_ADVANCE - except may overrun @_dst and @_src by
+ * COPYLENGTH:
+ *
+ * Note: src and dst may overlap (with src < dst) - we must do the copy in
+ * STEPSIZE chunks for correctness
+ *
+ * Note also: length may be negative - we must not call memcpy if length is
+ * negative, but still adjust dst and src by length
+ */
+#define MEMCPY_ADVANCE_CHUNKED(_dst, _src, _length) \
+do { \
+ u8 *_end = (_dst) + (_length); \
+ while ((_dst) < _end) \
+ LZ4_COPYPACKET(_src, _dst); \
+ _src -= (_dst) - _end; \
+ _dst = _end; \
+} while (0)
+
+#define MEMCPY_ADVANCE_CHUNKED_NOFIXUP(_dst, _src, _end)\
+do { \
+ while ((_dst) < (_end)) \
+ LZ4_COPYPACKET((_src), (_dst)); \
+} while (0)
+
+struct lz4_hashtable {
+#if LZ4_ARCH64
+ const u8 * const base;
+ u32 *table;
+#else
+ const int base;
+ const u8 *table;
+#endif
+};
+
+#if LZ4_ARCH64
+#define HTYPE u32
+#else /* 32-bit */
+#define HTYPE const u8*
+#endif
+
+#ifdef __BIG_ENDIAN
+#define LZ4_NBCOMMONBYTES(val) (__builtin_clzl(val) >> 3)
+#else
+#define LZ4_NBCOMMONBYTES(val) (__builtin_ctzl(val) >> 3)
+#endif
+
+static inline unsigned common_length(const u8 *l, const u8 *r,
+ const u8 *const l_end)
+{
+ const u8 *l_start = l;
+
+ while (likely(l <= l_end - sizeof(long))) {
+ unsigned long diff =
+ get_unaligned((unsigned long *) l) ^
+ get_unaligned((unsigned long *) r);
+
+ if (diff)
+ return l + LZ4_NBCOMMONBYTES(diff) - l_start;
+
+ l += sizeof(long);
+ r += sizeof(long);
+ }
+#if LZ4_ARCH64
+ if (l <= l_end - 4 && A32(r) == A32(l)) {
+ l += 4;
+ r += 4;
+ }
+#endif
+ if (l <= l_end - 2 && A16(r) == A16(l)) {
+ l += 2;
+ r += 2;
+ }
+ if (l <= l_end - 1 && *r == *l) {
+ l++;
+ r++;
+ }
+
+ return l - l_start;
+}
+
+static inline unsigned encode_length(u8 **op, unsigned length)
+{
+ if (length >= LENGTH_LONG) {
+ length -= LENGTH_LONG;
+
+ for (; length > 254 ; length -= 255)
+ *(*op)++ = 255;
+ *(*op)++ = length;
+ return LENGTH_LONG;
+ } else
+ return length;
+}
projects / bcachefs-tools-debian / commitdiff