#include "spadfs.h"

#define DECREASE_FREESPACE	0
#define INCREASE_FREESPACE	1

/* Helper function for GROUP_ACTION */

static __finline__ int GROUP_ACTION_INLINE(SPADFS *fs, unsigned group, unsigned len, int action)
{
	if (__likely(action == INCREASE_FREESPACE)) {
		if (__unlikely(fs->group_info[group].freespace + len > (__d_off)1 << fs->sectors_per_group_bits)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOCATION COUNT FOR GROUP %u OVERFLOW: %LX, %X", group, (unsigned long long)fs->group_info[group].freespace, len);
			return -EFSERROR;
		}
		fs->group_info[group].freespace += len;
		fs->group_info[group].zone->freespace += len;
	} else {
		if (__unlikely(fs->group_info[group].freespace < len) || __unlikely(fs->group_info[group].zone->freespace < len)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOCATION COUNT FOR GROUP %u UNDERFLOW: %LX, %LX, %X", group, (unsigned long long)fs->group_info[group].freespace, (unsigned long long)fs->group_info[group].zone->freespace, len);
			return -EFSERROR;
		}
		fs->group_info[group].freespace -= len;
		fs->group_info[group].zone->freespace -= len;
	}
	return 0;
}

/* Helper function for GROUP_ACTION */

static int GROUP_ACTION_NOINLINE(SPADFS *fs, unsigned group, unsigned len, int action)
{
	return GROUP_ACTION_INLINE(fs, group, len, action);
}

/* Action is either DECREASE_FREESPACE or INCREASE_FREESPACE.
   This function will update in-memory group statistics (and handle correctly
   situations like alloc or free crossing multiple groups).
*/

static int GROUP_ACTION(SPADFS *fs, __d_off start, unsigned len, int action)
{
	unsigned start_group, end_group;
	start_group = start >> fs->sectors_per_group_bits;
	end_group = (start + (len - 1)) >> fs->sectors_per_group_bits;
	if (__unlikely(end_group >= fs->n_groups)) {
range_error:
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "GROUP_ACTION: INVALID RANGE (%LX,%X)", (unsigned long long)start, len);
		return -EFSERROR;
	}
	if (__unlikely(start_group > end_group)) {
		goto range_error;
	}
	if (__likely(start_group == end_group)) {
		return GROUP_ACTION_INLINE(fs, start_group, len, action);
	} else {
		int r = GROUP_ACTION_NOINLINE(fs, start_group, fs->group_mask + 1 - (start & fs->group_mask), action);
		if (__unlikely(r))
			return r;
		for (start_group++; start_group < end_group; start_group++) {
			r = GROUP_ACTION_NOINLINE(fs, start_group, fs->group_mask + 1, action);
			if (__unlikely(r))
				return r;
		}
		r = GROUP_ACTION_NOINLINE(fs, start_group, ((start + len - 1) & fs->group_mask) + 1, action);
		return r;
	}
}

/* Convert pointer to mapped apage to index of an apage on filesystem
   --- for error prints
*/

static int MAP_2_APAGE_N(SPADFS *fs, APAGE_MAP *map)
{
	unsigned i, j;
	if (map == fs->tmp_map)
		return -1;
	for (i = 0; i < fs->n_apages; i++)
		for (j = 0; j < 2; j++)
			if (fs->apage_info[i].map[j] == map)
				return i;
	panic("MAP_2_APAGE_N: INVALID POINTER %p", map);
}

/* Get 16-byte entry in mapped apage */

static inline struct aentry *GET_AENTRY(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	if (__unlikely(off & (sizeof(struct aentry) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "UNALIGNED APAGE ENTRY %u, APAGE %d", off, MAP_2_APAGE_N(fs, map));
		return ERR_PTR(-EFSERROR);
	}
	if (__unlikely(off >= fs->apage_size)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE ENTRY %u TOO BIG, APAGE %d", off, MAP_2_APAGE_N(fs, map));
		return ERR_PTR(-EFSERROR);
	}
	return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_block_bits)].entry + (off & (fs->blocksize - 1)));
}

/* Get 16-byte entry in mapped apage assuming that the index is valid */

static inline struct aentry *GET_AENTRY_VALID(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	if (__unlikely(off & (sizeof(struct aentry) - 1))
	 || __unlikely(off >= fs->apage_size)) {
		panic("GET_AENTRY_VALID: invalid offset %u", off);
	}
	return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_block_bits)].entry + (off & (fs->blocksize - 1)));
}

/* Get first 16-byte entry of an apage --- its head */

static inline struct apage_head *GET_HEAD(APAGE_MAP *map)
{
	return (struct apage_head *)map[0].entry;
}

static void FREE_APAGE_BUFFERS(APAGE_MAP *map, unsigned n)
{
	while (n--)
		__brelse(map++->bh);
}

static int READ_APAGE_BUFFERS(SPADFS *fs, APAGE_MAP *map, __d_off sec, unsigned n)
{
	unsigned i;
	for (i = 0; i < n; i++) {
		if (__unlikely(IS_ERR(map[i].entry = SPADFS_READ_SECTOR(fs, sec, &map[i].bh, "READ_APAGE_BUFFERS")))) {
			int r = PTR_ERR(map[i].entry);
			FREE_APAGE_BUFFERS(map, i);
			return r;
		}
		sec += 1U << fs->sectors_per_block_bits;
	}
	return 0;
}

#define invalid_flags(flags, fs)	\
	__unlikely(((flags) & (APAGE_SIZE_BITS | APAGE_BLOCKSIZE_BITS)) != ((((fs)->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | ((fs)->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT)))

/* Map existing apage apage
   ap is index of an apage
   flags is one of
	MAP_READ --- map apage for read
	MAP_NEW --- map yet unused apage
	MAP_WRITE --- map apage for write (i.e. fixup cc/txc and copy its
		content if its the first map since last sync)
	MAP_ALLOC --- like MAP_WRITE but skip completely full apages
		(an optimization)
*/

#define MAP_READ	0
#define MAP_NEW		1
#define MAP_WRITE	2
#define MAP_ALLOC	3

static APAGE_MAP *MAP_APAGE(SPADFS *fs, unsigned ap, int flags, APAGE_MAP **other)
{
	struct apage_info *info;
	unsigned idx, i;
	int r;
	__d_off sec;
	struct apage_head *a;
	struct apage_head *aa;
	sec = fs->apage_index[ap].apage;
	if (__unlikely((unsigned)sec & ((1U << fs->sectors_per_page_bits) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "MISALIGNED APAGE %u AT BLOCK %LX", ap, (unsigned long long)sec);
		return ERR_PTR(-EFSERROR);
	}
	info = &fs->apage_info[ap];
	if (__unlikely(r = READ_APAGE_BUFFERS(fs, info->map[0], sec, fs->n_apage_mappings))) {
		return ERR_PTR(r);
	}
	if (__unlikely(r = READ_APAGE_BUFFERS(fs, info->map[1], sec + (1 << (fs->sectors_per_page_bits - 1)), fs->n_apage_mappings))) {
		FREE_APAGE_BUFFERS(info->map[0], fs->n_apage_mappings);
		return ERR_PTR(r);
	}

	a = GET_HEAD(info->map[0]);
	if (__unlikely(a->magic != APAGE_MAGIC)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BAD MAGIC ON APAGE %LX", (unsigned long long)sec);
		r = -EFSERROR;
		goto ret_err;
	}

	idx = !CC_VALID(fs, a->cc, a->txc);

	if (__unlikely(flags == MAP_NEW)) {
		goto read_ok;
	}

	if (invalid_flags(GET_HEAD(info->map[idx])->s.u.l.flags, fs)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE %LX HAS INVALID FLAGS %02X", (unsigned long long)sec, GET_HEAD(info->map[idx])->s.u.l.flags);
		r = -EFSERROR;
		goto ret_err;
	}

	if (__unlikely(flags == MAP_READ)) {
read_ok:
		if (__likely(other != NULL)) {
			*other = info->map[idx ^ 1];
		} else {
			FREE_APAGE_BUFFERS(info->map[idx ^ 1], fs->n_apage_mappings);
		}
		return info->map[idx];
	}
	if (__likely(flags == MAP_ALLOC)) {
		aa = GET_HEAD(info->map[idx]);
		if (__unlikely(!(aa->s.u.l.flags & APAGE_BITMAP)) && __unlikely(!aa->s.u.l.last)) {
			r = -ENOSPC;
			goto ret_err;
		}
	}
	if (__likely(CC_CURRENT(fs, a->cc, a->txc))) {
		if (invalid_flags(GET_HEAD(info->map[idx ^ 1])->s.u.l.flags, fs)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE %LX HAS INVALID FLAGS %02X IN INACTIVE PART", (unsigned long long)sec, GET_HEAD(info->map[idx ^ 1])->s.u.l.flags);
			r = -EFSERROR;
			goto ret_err;
		}
		goto read_ok;
	}
	start_atomic_buffer_modify(info->map[0][0].bh);
	CC_SET_CURRENT(fs, &a->cc, &a->txc);
	end_atomic_buffer_modify(info->map[0][0].bh);

	for (i = 0; i < fs->n_apage_mappings; i++) {
		char *to, *from;
		unsigned size;
		to = (char *)info->map[idx ^ 1][i].entry;
		from = (char *)info->map[idx][i].entry;
		size = fs->blocksize;
		if (!i) {
			if (size > fs->apage_size)
				size = fs->apage_size;
			to += sizeof(struct apage_head) - sizeof(struct apage_subhead);
			from += sizeof(struct apage_head) - sizeof(struct apage_subhead);
			size -= sizeof(struct apage_head) - sizeof(struct apage_subhead);
		}
		memcpy(to, from, size);
		mark_buffer_dirty(info->map[idx ^ 1][i].bh);
	}
	idx ^= 1;
	goto read_ok;

ret_err:
	FREE_APAGE_BUFFERS(info->map[0], fs->n_apage_mappings);
	FREE_APAGE_BUFFERS(info->map[1], fs->n_apage_mappings);
	return ERR_PTR(r);
}

/* Unmap existing apage --- a companion to MAP_APAGE
   If the apage has been modified, "modified" argument must be nonzero
*/

static void UNMAP_APAGE(SPADFS *fs, APAGE_MAP *map, int modified)
{
	BUG_ON(!atomic_read(&map[0].bh->b_count));
	if (modified) {
		unsigned i;
		GET_HEAD(map)->s.u.l.flags &= ~APAGE_CHECKSUM_VALID;
		for (i = 0; i < fs->n_apage_mappings; i++)
			mark_buffer_dirty(map[i].bh);
	}
	FREE_APAGE_BUFFERS(map, fs->n_apage_mappings);
}

/* Copy one mapped apage to another */

static void COPY_APAGE(SPADFS *fs, APAGE_MAP *dst, APAGE_MAP *src)
{
	unsigned i;
	unsigned n = fs->n_apage_mappings;
	unsigned size = fs->blocksize;
	if (__unlikely(fs->apage_size < size))
		size = fs->apage_size;
	for (i = 0; i < n; i++)
		memcpy(dst[i].entry, src[i].entry, size);
}

/* Create apage that has all entries free (i.e. it doesn't specify any free
   space yet */

static void MAKE_APAGE(SPADFS *fs, APAGE_MAP *map)
{
	unsigned i;
	struct apage_head *head;
	struct aentry *ae;
	head = GET_HEAD(map);
	head->s.u.l.flags = ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | (fs->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT);
	head->s.u.l.checksum = 0;
	head->s.u.l.freelist = sizeof(struct aentry);
	head->s.u.l.last = 0;
	head->s.u.l.first = 0;
	i = sizeof(struct aentry);
	do {
		ae = GET_AENTRY_VALID(fs, map, i);
		ae->start = 0;
		ae->len = 0;
		ae->prev = 0;
		ae->next = i += sizeof(struct aentry);
	} while (i < fs->apage_size);
	ae->next = 0;
}

/* Create bitmap apage that has all bits marked as "allocated" */

static void MAKE_APAGE_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start)
{
	unsigned i;
	struct apage_head *head;
	struct aentry *ae;
	head = GET_HEAD(map);
	head->s.u.b.flags = ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | (fs->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT) | APAGE_BITMAP;
	head->s.u.b.checksum = 0;
	head->s.u.b.start1 = MAKE_PART_1(start);
	head->s.u.b.start0 = MAKE_PART_0(start);
	i = sizeof(struct aentry);
	do {
		ae = GET_AENTRY_VALID(fs, map, i);
		memset(ae, 0xff, sizeof(struct aentry));
		i += sizeof(struct aentry);
	} while (i < fs->apage_size);
}

/* Helper for SPADFS_COUNT_FREE_SPACE
   Count free space in one apage */

static int GET_MAP_STATS(SPADFS *fs, APAGE_MAP *map, __d_off *freespace)
{
	struct apage_head *head = GET_HEAD(map);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		unsigned p = head->s.u.l.first;
		unsigned max_loop = (512 / sizeof(struct aentry) / 2) << fs->sectors_per_page_bits;
		while (p) {
			struct aentry *aentry = GET_AENTRY(fs, map, p);
			if (__unlikely(IS_ERR(aentry)))
				return PTR_ERR(aentry);
			if (__unlikely(GROUP_ACTION(fs, aentry->start, aentry->len, INCREASE_FREESPACE)))
				return -EFSERROR;
			*freespace += aentry->len;
			p = aentry->next;
			if (__unlikely(!--max_loop)) {
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "GET_MAP_STATS: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
				return -EFSERROR;
			}
		}
		return 0;
	} else {
		unsigned p;
		__d_off bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1);
		for (p = sizeof(struct aentry); p < fs->apage_size; p += sizeof(struct aentry)) {
			unsigned i;
			__u8 *bm = (char *)GET_AENTRY(fs, map, p);
			if (__unlikely(IS_ERR(bm)))
				return PTR_ERR(bm);
			for (i = 0; i < sizeof(struct aentry) * 8; i++) {
				if (BITMAP_TEST_32_FULL(bm - sizeof(struct apage_head), i)) {
					i += 31;
					continue;
				}
				if (!BITMAP_TEST(bm - sizeof(struct apage_head), i)) {
					*freespace += fs->blocksize / 512;
					if (__unlikely(GROUP_ACTION(fs, bst + ((i + (p - sizeof(struct aentry)) * 8) << fs->sectors_per_block_bits), fs->blocksize / 512, INCREASE_FREESPACE)))
						return -EFSERROR;
				}
			}
		}
		return 0;
	}
}

/* Counts free space during mount */

int SPADFS_COUNT_FREE_SPACE(SPADFS *fs)
{
	unsigned i;
	__d_off freespace = 0;
	for (i = 0; i < fs->n_active_apages; i++) {
		int r;
		APAGE_MAP *map;
		if (__unlikely(IS_ERR(map = MAP_APAGE(fs, i, MAP_READ, NULL))))
			return PTR_ERR(map);
		if (__unlikely(r = GET_MAP_STATS(fs, map, &freespace)))
			return r;
		UNMAP_APAGE(fs, map, 0);
	}
	fs->freespace = freespace;

	freespace = 0;
	for (i = 0; i < fs->n_groups; i++) {
		freespace += fs->group_info[i].freespace;
	}
	if (__unlikely(freespace != fs->freespace)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE SPACE IN GROUPS MISCOUNTED, %LX != %LX", (unsigned long long)freespace, (unsigned long long)fs->freespace);
		return -EFSERROR;
	}

	freespace = 0;
	for (i = 0; i < 3; i++) {
		freespace += fs->zones[i].freespace;
	}
	if (__unlikely(freespace != fs->freespace)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE SPACE IN ZONES MISCOUNTED, %LX != %LX", (unsigned long long)freespace, (unsigned long long)fs->freespace);
		return -EFSERROR;
	}


	return 0;
}

/* Find the last apage entry mapping area before the specified block
   Uses algorithm with average O(sqrt(n)) complexity:
	Scan any sqrt(n) blocks (the blocks are selected deliberately in
		64-byte chunks (4 entries, each 16 bytes) to minimize cache
		pollution)
	From scanned blocks, select the one with highest start that is lower
		than "before"
	From the selected block, scan forward using double-linked list pointers
		(on average we scan sqrt(n) apage entries in this pass)
 */

static int FIND_BLOCK_BEFORE(SPADFS *fs, APAGE_MAP *map, __d_off before)
{
	unsigned i;
	unsigned rlimit;
	__d_off xst = 0;
	unsigned st = 0, nx;
	struct aentry *sta = (struct aentry *)GET_HEAD(map), *nxa;
	for (i = 4 * sizeof(struct aentry); i <= fs->apage_size - 4 * sizeof(struct aentry); i += 124 * sizeof(struct aentry)) {
		unsigned j;
		for (j = 0; j < 4; j++) {
			struct aentry *a = GET_AENTRY_VALID(fs, map, i);
			if (__unlikely(IS_ERR(a)))
				return PTR_ERR(a);
			if (a->len && a->start <= before && __unlikely(a->start >= xst))
				xst = a->start, st = i, sta = a;
			i += sizeof(struct aentry);
		}
	}
	rlimit = fs->apage_size;
	do {
		nx = sta->next;
		nxa = GET_AENTRY(fs, map, nx);
		if (__unlikely(IS_ERR(nxa)))
			return PTR_ERR(nxa);
		if (__unlikely(!nx) || nxa->start > before)
			return st;
		st = nx;
		sta = nxa;
	} while (__likely(rlimit -= sizeof(struct aentry)));
	SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FIND_BLOCK_BEFORE: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
	return -EFSERROR;
}

/* Simple operations with bits in bitmaps */

static inline int BMP_TEST(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return (p[(off % (sizeof(struct aentry) * 8)) >> 5] >> (off & 31)) & 1;
}

static inline int BMP_SET(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] |= 1 << (off & 31);
}

static inline int BMP_CLEAR(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] &= ~(1 << (off & 31));
}

static inline int BMP_TEST_32_FULL(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] == 0xffffffffu;
}

/* Helper for CHECK_OTHER_MAP */

static int CHECK_OTHER_MAP_BMP(SPADFS *fs, APAGE_MAP *map, __d_off off, unsigned n_sec, __d_off *next_free)
{
	struct apage_head *head = GET_HEAD(map);
	unsigned Xboff = BITMAP_OFFSET(head, off);
	unsigned Xn_sec = BITMAP_LEN(head, n_sec);
	if (__unlikely(Xboff + Xn_sec <= Xboff) || __unlikely(Xboff + Xn_sec > BITMAP_SIZE(fs->apage_size))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "CHECK_OTHER_MAP_BMP: BITMAP ERROR, APAGE %d, XBOFF %X, OFF %LX, N_SEC %X", MAP_2_APAGE_N(fs, map), Xboff, (unsigned long long)off, n_sec);
		return -EFSERROR;
	}
	do {
		if (__unlikely(BMP_TEST(fs, map, Xboff))) {
			while (++Xboff < BITMAP_SIZE(fs->apage_size))
				if (!BMP_TEST(fs, map, Xboff)) {
					*next_free = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1) + (Xboff << fs->sectors_per_block_bits);
					return -EBUSY;
				}
			return -EBUSY;
		}
		Xboff++;
	} while (--Xn_sec);
	return 0;
}

/* When we are allocating space, we must make sure that we don't allocate space
   that is free but that would be valid in case of crash --- this functions
   checks it
*/

static int CHECK_OTHER_MAP(SPADFS *fs, APAGE_MAP *map, __d_off off, unsigned n_sec, __d_off *next_free)
{
	if (__unlikely(!map))
		return 0;
	*next_free = 0;
	if (__likely(!(GET_HEAD(map)->s.u.l.flags & APAGE_BITMAP))) {
		int preblk, postblk;
		struct aentry *e;
		preblk = FIND_BLOCK_BEFORE(fs, map, off);
		if (__unlikely(preblk < 0))
			return preblk;
		e = GET_AENTRY_VALID(fs, map, preblk);
		if (__likely(preblk != 0)) {
			if (off + n_sec <= e->start + e->len) {
				return 0;
			}
		}
		postblk = e->next;
		if (__likely(postblk != 0)) {
			e = GET_AENTRY(fs, map, postblk);
			if (__unlikely(IS_ERR(e)))
				return PTR_ERR(e);
			*next_free = e->start;
		}
		return -EBUSY;
	} else {
		return CHECK_OTHER_MAP_BMP(fs, map, off, n_sec, next_free);
	}
}

/* Helper for ALLOC_BLOCKS_IN_APAGE --- do allocation in case it is bitmap */

static int ALLOC_BLOCKS_IN_APAGE_BMP(SPADFS *fs, APAGE_MAP *map, APAGE_MAP *other, struct alloc *al)
{
	struct apage_head *head = GET_HEAD(map);
	unsigned shift = fs->sectors_per_block_bits;
	unsigned Xbmax = BITMAP_SIZE(fs->apage_size);
	unsigned Xboff;
	__d_off bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1);
	__d_off bottom, top;
	bottom = al->sector;
	if (bottom < bst)
		bottom = bst;
	top = al->top;
	if (__likely(top > bst + (Xbmax << shift)))
		top = bst + (Xbmax << shift);
	if (__unlikely(bottom >= top))
		return -ENOSPC;
	Xboff = (bottom - bst) >> shift;
	if (top - bst < (Xbmax << shift))
		Xbmax = (top - bst) >> shift;
	while (Xboff < Xbmax) {
		if (__likely(BMP_TEST_32_FULL(fs, map, Xboff))) {
			Xboff = (Xboff + 32) & ~31;
			continue;
		}
test_bit:
		if (__unlikely(!BMP_TEST(fs, map, Xboff))) {
			__d_off new_bot;
			unsigned Xc;
			if (__unlikely(al->flags & ALLOC_METADATA)) {
				if ((((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags)) + al->n_sectors > ALLOC_MASK(al->flags) + 1)
					goto align;
			} else if (((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags)) {
				unsigned Xplus;
align:
				Xplus = (ALLOC_MASK(al->flags) + 1 - (((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags))) >> shift;
				if (__unlikely(!Xplus))
					Xplus = 1;
				Xboff += Xplus;
				continue;
			}
			for (Xc = 1; Xc < al->n_sectors >> shift; Xc++)
				if (__unlikely(Xboff + Xc >= Xbmax) || BMP_TEST(fs, map, Xboff + Xc)) {
					Xboff += Xc + 1;
					goto cont;
				}
			if (__unlikely(CHECK_OTHER_MAP(fs, other, bst + (Xboff << shift), al->n_sectors, &new_bot))) {
				if (__likely(new_bot > bst + (Xboff << shift)) && __likely(new_bot < bst + (Xbmax << shift))) {
					Xboff = (new_bot - bst + ((1 << shift) - 1)) >> shift;
					continue;
				}
				return -ENOSPC;
			}
			al->sector = bst + (Xboff << shift);
			for (Xc = 0; Xc < al->n_sectors >> shift; Xc++) BMP_SET(fs, map, Xboff + Xc);
			return 0;
		}
		Xboff++;
		if (__likely(Xboff < Xbmax) && __likely(Xboff & 31))
			goto test_bit;
		cont:;
	}
	return -ENOSPC;
}

/* Delete one aentry from double linked list and add it to single linked
   freelist */

static inline void DELETE_BLOCK(SPADFS *fs, APAGE_MAP *map, struct aentry *e)
{
	struct aentry *pre, *post;
	struct apage_head *head;
	unsigned nblk;
	pre = GET_AENTRY(fs, map, e->prev);
	if (__unlikely(IS_ERR(pre)))
		return;
	pre->next = e->next;
	post = GET_AENTRY(fs, map, e->next);
	if (__unlikely(IS_ERR(post)))
		return;
	nblk = post->prev;
	post->prev = e->prev;
	e->start = 0;
	e->len = 0;
	e->prev = 0;
	head = GET_HEAD(map);
	e->next = head->s.u.l.freelist;
	head->s.u.l.freelist = nblk;
}

/* Alloc block in a given aentry */

static inline int ALLOC_BLOCK(SPADFS *fs, struct aentry *a, struct alloc *al, APAGE_MAP *other)
{
	__d_off bottom, top, new_bot;
	bottom = al->sector;
	if (__likely(bottom < a->start))
		bottom = a->start;
	top = al->top;
	if (__likely(top > a->start + a->len))
		top = a->start + a->len;
new_bottom:
	if (__unlikely(al->flags & ALLOC_METADATA))
		if (__likely(((unsigned)bottom & ALLOC_MASK(al->flags)) + al->n_sectors <= ALLOC_MASK(al->flags) + 1))
			goto skip_pad;
	bottom = (bottom + ALLOC_MASK(al->flags)) & ~(__d_off)ALLOC_MASK(al->flags);
skip_pad:
	if (__unlikely(bottom + al->n_sectors > top))
		return -ENOSPC;

	if (__unlikely(CHECK_OTHER_MAP(fs, other, bottom, al->n_sectors, &new_bot))) {
		if (__likely(new_bot > bottom)) {
			bottom = new_bot;
			goto new_bottom;
		}
		return -ENOSPC;
	}
	if (__unlikely(bottom > a->start)) {
		al->flags |= ALLOC_FREE_FROM;
		al->top = a->start;
	}
	a->len -= ((__u32)bottom - (__u32)a->start) + al->n_sectors;
	a->start = bottom + al->n_sectors;
	al->sector = bottom;
	return 0;
}

/* Alloc block run in a given apage */

static int ALLOC_BLOCKS_IN_APAGE(SPADFS *fs, APAGE_MAP *map, APAGE_MAP *other, struct alloc *al)
{
	struct apage_head *head = GET_HEAD(map);
	if (__unlikely(IS_ERR(head)))
		return PTR_ERR(head);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		struct aentry *e;
		int n_cycles;
		int blk = FIND_BLOCK_BEFORE(fs, map, al->sector);
		if (__unlikely(blk < 0))
			return blk;
		e = GET_AENTRY_VALID(fs, map, blk);
		n_cycles = fs->apage_size;
next_try:
		if (!blk || al->sector >= e->start + e->len) {
			blk = e->next;	/* valid from FIND_BLOCK_BEFORE */
			if (__unlikely(!blk))
				return -ENOSPC;
			e = GET_AENTRY_VALID(fs, map, blk);
		}
		if (__unlikely(e->start >= al->top))
			return -ENOSPC;
		if (e->len >= al->n_sectors && __likely(!ALLOC_BLOCK(fs, e, al, other))) {
			if (__unlikely(!e->len))
				DELETE_BLOCK(fs, map, e);
			return 0;
		}
		blk = e->next;
		if (__unlikely(!blk))
			return -ENOSPC;
		e = GET_AENTRY(fs, map, blk);
		if (__unlikely(blk < 0))
			return blk;
		if (__likely(n_cycles -= sizeof(struct aentry)))
			goto next_try;
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOC_BLOCKS_IN_APAGE: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
		return -EFSERROR;
	} else {
		return ALLOC_BLOCKS_IN_APAGE_BMP(fs, map, other, al);
	}
}

/* Map apage and alloc block run in it */

static __finline__ int TRY_APAGE(SPADFS *fs, unsigned ap, struct alloc *al)
{
	APAGE_MAP *map, *other;
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_ALLOC, &other))))
		return PTR_ERR(map);
	if (__likely(!ALLOC_BLOCKS_IN_APAGE(fs, map, other, al))) {
		UNMAP_APAGE(fs, map, 1);
		UNMAP_APAGE(fs, other, 0);
		return 0;
	}
	UNMAP_APAGE(fs, map, 0);
	UNMAP_APAGE(fs, other, 0);
	return -ENOSPC;
}

/* Convert block number to an index of apage --- use binary search */

static int ADDR_2_APAGE(SPADFS *fs, __d_off o, char *msg)
{
	int a1, a2, a;
	a2 = fs->n_active_apages - 1;
	a1 = 0;
again:
	a = (a1 + a2) >> 1;
	if (o < (__d_off)fs->apage_index[a - 1].end_block)
		a2 = a - 1;
	else if (o >= (__d_off)fs->apage_index[a].end_block)
		a1 = a + 1;
	else
		return a;
	if (__likely(a1 <= a2))
		goto again;
	SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "CAN'T FIND APAGE FOR BLOCK %LX, STUCK ON %d/%d, CALLED FROM %s", (unsigned long long)o, a1, a2, msg);
	return -EFSERROR;
}

/* Alloc block run in a range specified by al->sector and al->top */

static int ALLOC_BLOCKS_IN_RANGE(SPADFS *fs, struct alloc *al)
{
	int ap, tap;
	if (__unlikely(al->top > fs->size))
		al->top = fs->size;
	if (__unlikely(al->sector >= al->top))
		return -ENOSPC;
	ap = ADDR_2_APAGE(fs, al->sector, "ALLOC_BLOCKS_IN_RANGE 1");
	if (__unlikely(ap < 0))
		return ap;
	if (__likely(al->top <= fs->apage_index[ap].end_block))
		tap = ap;
	else {
		tap = ADDR_2_APAGE(fs, al->top - 1, "ALLOC_BLOCKS_IN_RANGE 2");
		if (__unlikely(tap < 0))
			return tap;
	}
next_ap:
	if (__likely(!TRY_APAGE(fs, ap, al)))
		return 0;
	ap++;
	if (__unlikely(ap <= tap))
		goto next_ap;
	return -ENOSPC;
}

/* Get number of consecutive free blocks at given location
   "max" is the limit, don't scan past this, even if there are more free blocks
*/

static unsigned GET_BLOCKLEN_AT(SPADFS *fs, __d_off block, unsigned max)
{
	struct apage_head *head;
	APAGE_MAP *map;
	int ret;
	int ap = ADDR_2_APAGE(fs, block, "GET_BLOCKLEN_AT");
	if (__unlikely(ap < 0))
		return 0;
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_READ, NULL))))
		return 0;
	head = GET_HEAD(map);
	if (__unlikely(IS_ERR(head)))
		goto unmap_0;
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		struct aentry *e;
		int blk = FIND_BLOCK_BEFORE(fs, map, block);
		if (__unlikely(blk < 0))
			goto unmap_0;
		e = GET_AENTRY_VALID(fs, map, blk);
		if (block >= e->start + e->len)
			goto unmap_0;
		ret = e->start + e->len - block;
		if (__likely(ret > max))
			ret = max;
unmap_ret:
		UNMAP_APAGE(fs, map, 0);
		return ret;
	} else {
		unsigned Xboff = BITMAP_OFFSET(head, block);
		unsigned add = fs->blocksize / 512;
		ret = 0;
		while (ret < max && __likely(Xboff < BITMAP_SIZE(fs->apage_size)) && !BMP_TEST(fs, map, Xboff))
			ret += add, Xboff++;
		goto unmap_ret;

	}

unmap_0:
	UNMAP_APAGE(fs, map, 0);
	return 0;
}

static int ALLOC_BLOCKS_IN_DIFFERENT_GROUP(SPADFNODE *f, struct alloc *al);

/* This is the main entry to block allocator. May be called by any other
   subsystem.
   In order to minimize CPU usage by passing all allocation parameters between
   allocator functions, we pass pointer to struct alloc instead. struct alloc is
   usually placed on caller's stack.
   The following fields in "struct alloc" must be filled before calling this
   function:
	sector: a hint where the allocation should start. Must be dividable by
		"sectors_per_block" value
	n_sectors: preferred number of sectors. Must be dividable by
		"sectors_per_block" value
	flags:
		ALLOC_METADATA: we are allocating metadata, this means:
			- prefer metadata zone
			- alloc exactly n_sectors or nothing (the caller must
			handle failure of allocating more than 1 block without
			loss of functionality --- i.e. create chains instead of
			hash tables in directories)
			- n_sectors must be less or equal than
			"sectors_per_page"
			- the resulting allocation won't cross
			"sectors_per_page" boundary
		ALLOC_SMALL_FILE: we are allocating small file (below threshold)
			- prefer small file zone
			- may alloc less blocks if the disk is too fragmented,
			the resulting number is returned in n_sectors. The
			result is always multiple of "sectors_per_block"
		ALLOC_BIG_FILE: we are allocating big file
			- prefer big file zone
			- prefer allocating at sectors_per_cluster boundary
			- may alloc less blocks if the disk is too fragmented,
			the resulting number is returned in n_sectors. The
			result is always multiple of "sectors_per_block"
		ALLOC_PARTIAL_AT_GOAL: may be ORed with ALLOC_SMALL_FILE or
			ALLOC_BIG_FILE. It means that if the block at al->sector
			is free, we always allocate from al->sector, even if we
			allocate less than n_sectors sectors --- this is used
			when extending file with non-zero length
   Returns:
   	-ENOSPC --- the blocks couldn't be allocated
		If ALLOC_SMALL_FILE or ALLOC_BIG_FILE was specified, it means
		that the disk is totally full
		If ALLOC_METADATA was specified and n_sectors ==
		"sectors_per_page", it also means that the disk is totally full
		If ALLOC_METADATA was specified and n_sectors is larger, it
		means that the disk is too fragmented to allocate specified
		number of blocks
	-EIO, -EFSERROR, -E... --- some error occured
	0 --- succesful, then
		al->sector --- allocated sector
		al->n_sectors --- allocated number of sectors
		al->flags & ALLOC_NEW_GROUP_HINT --- the caller should set group
			hint for the file's directory to returned value
*/

int SPADFS_ALLOC_BLOCKS(SPADFNODE *f, struct alloc *al)
{
	SPADFS *fs = f->fs;
	down(&fs->alloc_lock);

	if (__unlikely(al->n_sectors & ((1 << fs->sectors_per_block_bits) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "TRYING TO ALLOCATE %u BLOCKS", al->n_sectors);
		up(&fs->alloc_lock);
		return -EFSERROR;
	}
	al->sector &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);

	if (__unlikely(al->sector >= fs->size))
		al->sector = 0;

	if (__unlikely(al->flags & ALLOC_METADATA)) {
		al->flags |= ((1 << fs->sectors_per_page_bits) - 1) * ALLOC_MASK_1;
	}
	if (al->flags & ALLOC_BIG_FILE) {
		al->flags |= ((1 << fs->sectors_per_cluster_bits) - 1) * ALLOC_MASK_1;
		al->sector &= ~(__d_off)((1 << fs->sectors_per_cluster_bits) - 1);
	}

	if (al->flags & ALLOC_PARTIAL_AT_GOAL) {
		__u32 bl = GET_BLOCKLEN_AT(fs, al->sector, al->n_sectors);
		if (__likely(bl != 0)) {
			unsigned orig_n_sectors = al->n_sectors;
			al->n_sectors = bl;
			while (1) {
				al->n_sectors &= ~(fs->blocksize / 512 - 1);
				if (__unlikely(!al->n_sectors))
					break;
				al->top = al->sector + al->n_sectors;
				if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al)))
					goto ok_allocated;
				al->n_sectors >>= 1;
			}
			al->n_sectors = orig_n_sectors;
		}
	}

	al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask;

	if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
ok_allocated:
		if (__unlikely(al->flags & ALLOC_FREE_FROM)) {
			int r;
			if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, al->top, al->sector - al->top, 0))) {
				up(&fs->alloc_lock);
				return r;
			}
		}
		if (__unlikely(al->n_sectors > fs->freespace)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE BLOCK COUNT UNDERRUN: %X < %X", (unsigned)fs->freespace, al->n_sectors);
			up(&fs->alloc_lock);
			return 0;
		}
		fs->freespace -= al->n_sectors;
		GROUP_ACTION(fs, al->sector, al->n_sectors, DECREASE_FREESPACE);
		up(&fs->alloc_lock);
		return 0;
	}
	if (__likely(!ALLOC_BLOCKS_IN_DIFFERENT_GROUP(f, al)))
		goto ok_allocated;
	up(&fs->alloc_lock);
	return -ENOSPC;
}

/* Average number of free blocks in group in zone. We couldn't divide 64-bit
   values in kernel, so get approximate value. */

static __finline__ __d_off ZONE_AVERAGE_FREE(struct spadfszone *z)
{
	unsigned n = 0;
	do {		/* !!! TODO: use fls ? */
		n++;
	} while (1 << n < z->grp_n);
	return z->freespace >> n;
}

/* A "slow path" helper for SPADFS_ALLOC_BLOCKS */

static int ALLOC_BLOCKS_IN_DIFFERENT_GROUP(SPADFNODE *f, struct alloc *al)
{
	SPADFS *fs = f->fs;
	__d_off orig_start = al->sector;
	struct spadfszone *z;
retry_less:
	if (__likely(al->flags & ALLOC_BIG_FILE)) {
		z = &fs->zones[2];
	} else if (__likely(al->flags & ALLOC_SMALL_FILE)) {
		z = &fs->zones[1];
	} else {
		z = &fs->zones[0];
	}
forward_quad_search:
	if (__likely(z->grp_n)) {
		unsigned pass;
		__d_off avg_free = ZONE_AVERAGE_FREE(z);
		if (__unlikely(!avg_free))
			avg_free = al->n_sectors;
retry:
		for (pass = 0; pass < 2; pass++) {
			unsigned i;
			unsigned grp_current = orig_start >> fs->sectors_per_group_bits;
			if (__unlikely(grp_current < z->grp_start) || __unlikely(grp_current >= z->grp_start + z->grp_n))
				grp_current = z->grp_start;
			i = 1;
			do {
				grp_current += __likely(!pass) ? i : 1;
				if (__unlikely(grp_current >= z->grp_start + z->grp_n))
					grp_current -= z->grp_n;
				if (fs->group_info[grp_current].freespace >= avg_free) {
					al->sector = (__d_off)grp_current << fs->sectors_per_group_bits;
					al->top = (__d_off)(grp_current + 1) << fs->sectors_per_group_bits;
					if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
alloc_success_set_group:
						al->flags |= ALLOC_NEW_GROUP_HINT;
						return 0;
					}
				}
				if (__likely(!pass))
					i *= 2;
				else
					i++;
			} while (i <= z->grp_n);
		}
		if (avg_free > al->n_sectors) {
			avg_free = al->n_sectors;
			goto retry;
		}
	}
try_another_zone:
	if (__likely(al->flags & ALLOC_BIG_FILE)) {
		if (__likely(z == &fs->zones[2]))
			z = &fs->zones[1];
		else if (__likely(z == &fs->zones[1]))
			z = &fs->zones[0];
		else
			goto failed;
	} else if (__likely(al->flags & ALLOC_SMALL_FILE)) {
		if (__likely(z == &fs->zones[1]))
			z = &fs->zones[2];
		else if (__likely(z == &fs->zones[2]))
			z = &fs->zones[0];
		else
			goto failed;
	} else {
		if (__likely(z == &fs->zones[0]))
			z = &fs->zones[1];
		else if (__likely(z == &fs->zones[1]))
			z = &fs->zones[2];
		else
			return -ENOSPC;
	}
	if (z == &fs->zones[2])
		goto forward_quad_search;
	if (__likely(z->grp_n)) {
		unsigned grp_current;
		__d_off avg_free = ZONE_AVERAGE_FREE(z) / SPADFS_AVG_FREE_DIVIDE_OTHERZONE;
		if (__unlikely(!avg_free))
			avg_free = al->n_sectors;
retry2:
		grp_current = z->grp_start + z->grp_n;
		do {
			grp_current--;
			if (fs->group_info[grp_current].freespace >= avg_free) {
				al->sector = (__d_off)grp_current << fs->sectors_per_group_bits;
				al->top = (__d_off)(grp_current + 1) << fs->sectors_per_group_bits;
				if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
					goto alloc_success_set_group;
				}
			}
		} while (grp_current > z->grp_start);
		if (avg_free > al->n_sectors) {
			avg_free = al->n_sectors;
			goto retry2;
		}
	}
	goto try_another_zone;
failed:
	/*
	 * Try to allocate less sectors
	 */
	al->n_sectors = (al->n_sectors >> 1) & ~(fs->blocksize / 512 - 1);
	if (__unlikely(!al->n_sectors))
		return -ENOSPC;
	if (__likely(ALLOC_MASK(al->flags))) {
		if (__unlikely(al->n_sectors <= ALLOC_MASK(al->flags)))
			al->flags = (al->flags & ~(ALLOC_MASK_MASK | ALLOC_BIG_FILE)) | ALLOC_SMALL_FILE;
		else
			al->n_sectors &= ~ALLOC_MASK(al->flags);
	}
	
	al->sector = orig_start;
	al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask;
	if (!ALLOC_BLOCKS_IN_RANGE(fs, al))
		return 0;
	goto retry_less;
}

/* Free a given sector run in mapped apage */

static int APAGE_FREE(SPADFS *fs, APAGE_MAP *map, __d_off off, __u32 len)
{
	struct apage_head *head = GET_HEAD(map);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		int preblk, postblk, newblk;
		struct aentry *pre, *post, *new;
		preblk = FIND_BLOCK_BEFORE(fs, map, off);
		if (__unlikely(preblk < 0))
			return preblk;
		pre = GET_AENTRY_VALID(fs, map, preblk);
		postblk = pre->next;	/* already valid */
		post = GET_AENTRY_VALID(fs, map, postblk);
		if (__likely(preblk != 0)) {
			if ((__d_off)pre->start + pre->len == off) {
				if (__unlikely((__u32)(pre->len + len) < (__u32)len))
					goto nj1;
				if (__likely(postblk != 0)) {
					if (off + len == (__d_off)post->start) {
						if (__unlikely((__u32)(pre->len + len + post->len) < (__u32)post->len))
							goto nj2;
						pre->len += len + post->len;
						DELETE_BLOCK(fs, map, post);
						goto done;
					} else if (__unlikely(off + len > (__d_off)post->start)) {
						goto post_over;
					}
				}
nj2:
				pre->len += len;
				goto done;
			} else if (__unlikely((__d_off)pre->start + pre->len > off)) {
				postblk = preblk;
				post = pre;
				goto post_over;
			}
		}
nj1:
		if (__likely(postblk != 0)) {
			if (off + len == (__d_off)post->start) {
				if (__unlikely((__u32)(post->len + len) < (__u32)len))
					goto nj3;
				post->start = off;
				post->len += len;
				goto done;
			} else if (__unlikely(off + len > (__d_off)post->start)) {
post_over:
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE (%LX,%X) DOES OVERLAP WITH BLOCK (%LX,%X)", (unsigned long long)off, (unsigned)len, (unsigned long long)post->start, (unsigned)post->len);
				return -EFSERROR;
			}
		}
nj3:
		if (__unlikely(!(newblk = head->s.u.l.freelist)))
			return 1;
		new = GET_AENTRY(fs, map, newblk);
		if (__unlikely(IS_ERR(new)))
			return PTR_ERR(new);
		head->s.u.l.freelist = new->next;
		new->start = off;
		new->len = len;
		new->prev = preblk;
		new->next = postblk;
		pre->next = newblk;
		post->prev = newblk;

done:
		return 0;
	} else {
		unsigned bmpoff = BITMAP_OFFSET(head, off);
		len = BITMAP_LEN(head, len);
		if (__unlikely(bmpoff + len <= bmpoff) || __unlikely(bmpoff + len > BITMAP_SIZE(fs->apage_size))) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BAD BITMAP OFFSET: %LX,%X FREE(%LX,%X)", (unsigned long long)MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1), bmpoff, (unsigned long long)off, (unsigned)len);
			return -EFSERROR;
		}
		do {
			BMP_CLEAR(fs, map, bmpoff);
			bmpoff++;
		} while (--len);
		return 0;
	}
}

static int SPLIT_APAGE(SPADFS *fs, APAGE_MAP *map, int ap);
static int CONVERT_TO_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start, __d_off end);

/* General block-freeing routine.
   SPADFS_FREE_BLOCKS (an inline in spadfs.h) should be called instead of this
   --- it locks the semaphore and forces "acct" to 1.

   start and n_sectors is the extent. acct means that we should do free space
   accounting. The only situation where acct could be 0 is the call from
   SPADFS_ALLOC_BLOCKS (in rare case when SPADFS_ALLOC_BLOCKS needs to allocate
   blocks in the middle of one free extent, the extent must be split to two ---
   instead of complicating logic of SPADFS_ALLOC_BLOCKS, it allocates more space
   from the beginning of the extent and then calls SPADFS_FREE_BLOCKS_ with
   acct == 0)
*/

int SPADFS_FREE_BLOCKS_(SPADFNODE *f, __d_off start, __d_off n_sectors, int acct)
{
	int ap;
	int r;
	SPADFS *fs = f->fs;
	APAGE_MAP *map;
	if (__unlikely(!n_sectors)
	 || __unlikely(((unsigned)n_sectors & ((1 << fs->sectors_per_block_bits) - 1)) != 0)
	 || __unlikely(start <= SUPERBLOCK_SECTOR)
	 || __unlikely(start >= fs->size)
	 || __unlikely(start + n_sectors > fs->size)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "SPADFS_FREE_BLOCKS: %LX, %LX", (unsigned long long)start, (unsigned long long)n_sectors);
		return -EFSERROR;
	}
retry:
	if (__unlikely(n_sectors & ~(__d_off)0xffffffffu)) {
		/* max block size is 64k, that is 128 sectors. Keep n_sectors aligned */
		if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, start, 0xffffff80u, acct)))
			return r;
		start += 0xffffff80u;
		n_sectors -= 0xffffff80u;
		goto retry;
	}
retry_32:
	ap = ADDR_2_APAGE(fs, start, "SPADFS_FREE_BLOCKS_");
	if (__unlikely(ap < 0))
		return ap;
	if ((__d_off)fs->apage_index[ap].end_block < start + n_sectors) {
		__u32 ss = (__d_off)fs->apage_index[ap].end_block - start;
		if (__unlikely(!ss) || __unlikely(ss >= n_sectors)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "SPADFS_FREE_BLOCKS: %LX,%LX - APAGE %d, APAGE_END %LX", (unsigned long long)start, (unsigned long long)n_sectors, ap, (unsigned long long)fs->apage_index[ap].end_block);
			return -EFSERROR;
		}
		if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, start, ss, acct)))
			return r;
		start += ss;
		n_sectors -= ss;
		goto retry_32;
	}
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_WRITE, NULL))))
		return PTR_ERR(map);
free_again:
	r = APAGE_FREE(fs, map, start, n_sectors);
	if (__likely(r <= 0)) {
		UNMAP_APAGE(fs, map, 1);
		if (__likely(!r) && __likely(acct)) {
			fs->freespace += n_sectors;
			GROUP_ACTION(fs, start, n_sectors, INCREASE_FREESPACE);
		}
		return r;
	}
	if (__likely((__d_off)fs->apage_index[ap].end_block - (__d_off)fs->apage_index[ap - 1].end_block > BITMAP_SIZE(fs->apage_size) << fs->sectors_per_block_bits)) {
		if (__unlikely(r = SPLIT_APAGE(fs, map, ap)))
			return r;
		goto retry_32;
	}
	if (__unlikely(r = CONVERT_TO_BITMAP(fs, map, (__d_off)fs->apage_index[ap - 1].end_block, (__d_off)fs->apage_index[ap].end_block))) {
		UNMAP_APAGE(fs, map, 1);
		return r;
	}
	goto free_again;
}

static int WRITE_APAGE_INDEX(SPADFS *fs);

/* Split apage to two */

static int SPLIT_APAGE(SPADFS *fs, APAGE_MAP *map0, int ap)
{
	int r;
	unsigned i, n;
	struct aentry *ae;
	__d_off split_block;
	APAGE_MAP *map1;
	struct apage_index_entry aie;
	unsigned b_s;
	if (__unlikely(fs->n_active_apages >= fs->n_apages)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALL %d APAGES USED", fs->n_active_apages);
		r = -EFSERROR;
ret0:
		UNMAP_APAGE(fs, map0, 1);
		return r;
	}
	n = 0;
	ae = (struct aentry *)GET_HEAD(map0);
	for (i = fs->apage_size / (2 * sizeof(struct aentry)); i; i--) {
		n = ae->next;
		ae = GET_AENTRY(fs, map0, n);
		if (__unlikely(IS_ERR(ae))) {
			r = PTR_ERR(ae);
			goto ret0;
		}
	}
	if (__unlikely(!n))
		split_block = ((__d_off)fs->apage_index[ap - 1].end_block + (__d_off)fs->apage_index[ap].end_block) >> 1;
	else
		split_block = ae->start;
	split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	b_s = (BITMAP_SIZE(fs->apage_size) >> 1 << fs->sectors_per_block_bits);
	if (__unlikely(split_block - (__d_off)fs->apage_index[ap - 1].end_block < b_s)) {
		split_block = (__d_off)fs->apage_index[ap - 1].end_block + b_s;
		split_block += (1 << fs->sectors_per_block_bits) - 1;
		split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	}
	if (__unlikely((__d_off)fs->apage_index[ap].end_block - split_block < b_s)) {
		split_block = (__d_off)fs->apage_index[ap].end_block - b_s;
		split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	}
	COPY_APAGE(fs, fs->tmp_map, map0);
	map1 = MAP_APAGE(fs, fs->n_active_apages, MAP_NEW, NULL);
	if (__unlikely(IS_ERR(map1))) {
		r = PTR_ERR(map1);
		goto ret0;
	}
	MAKE_APAGE(fs, map0);
	MAKE_APAGE(fs, map1);
	for (i = sizeof(struct aentry); i < fs->apage_size; i += sizeof(struct aentry)) {
		__d_off st;
		struct aentry *ae = GET_AENTRY_VALID(fs, fs->tmp_map, i);
		if (__unlikely(!ae->len))
			continue;
		st = ae->start;
		if (st >= split_block) {
			r = APAGE_FREE(fs, map1, st, ae->len);
		} else if (st + ae->len <= split_block) {
			r = APAGE_FREE(fs, map0, st, ae->len);
		} else {
			r = APAGE_FREE(fs, map0, st, split_block - st);
			if (__likely(!r)) r = APAGE_FREE(fs, map1, split_block, st + ae->len - split_block);
		}
		if (__unlikely(r)) {
			if (r > 0)
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "OUT OF SPACE WHEN SPLITTING APAGE"),
				r = -EFSERROR;
			UNMAP_APAGE(fs, map1, 1);
			goto ret0;
		}
	}
	UNMAP_APAGE(fs, map0, 1);
	UNMAP_APAGE(fs, map1, 1);
	memcpy(&aie, &fs->apage_index[fs->n_active_apages], sizeof(struct apage_index_entry));
	memmove(&fs->apage_index[ap + 2], &fs->apage_index[ap + 1], (fs->n_active_apages - ap - 1) * sizeof(struct apage_index_entry));
	memcpy(&fs->apage_index[ap + 1], &aie, sizeof(struct apage_index_entry));
	fs->apage_index[ap + 1].end_block = (__d_off)fs->apage_index[ap].end_block;
	fs->apage_index[ap].end_block = split_block;
	fs->n_active_apages++;
	return WRITE_APAGE_INDEX(fs);
}

/* Write apage index after the split */

static int WRITE_APAGE_INDEX(SPADFS *fs)
{
	unsigned i, n;
	__d_off sec;
	if (__likely(!CC_CURRENT(fs, fs->a_cc, fs->a_txc))) {
		struct txblock *tx;
		struct buffer_head *bh;
		if (__unlikely(IS_ERR(tx = SPADFS_READ_TX_BLOCK(fs, &bh, "WRITE_APAGE_INDEX 1"))))
			return PTR_ERR(tx);
		start_atomic_buffer_modify(bh);
		CC_SET_CURRENT(fs, &fs->a_cc, &fs->a_txc);
		tx->a_cc = fs->a_cc;
		tx->a_txc = fs->a_txc;
		SPADFS_TX_BLOCK_CHECKSUM(tx);
		end_atomic_buffer_modify(bh);
		__brelse(bh);
	}
	n = APAGE_INDEX_SECTORS(fs->n_apages, fs->blocksize);
	sec = CC_VALID(fs, fs->a_cc, fs->a_txc) ? fs->apage_index0_sec : fs->apage_index1_sec;
	for (i = 0; i < n; i += 1 << fs->sectors_per_block_bits, sec += 1 << fs->sectors_per_block_bits) {
		struct buffer_head *bh;
		void *p = SPADFS_GET_NEW_SECTOR(fs, sec, &bh, "WRITE_APAGE_INDEX 2");
		if (__unlikely(IS_ERR(p)))
			continue;
		memcpy(p, (__u8 *)fs->apage_index + (i << 9), fs->blocksize);
		mark_buffer_dirty(bh);
		brelse(bh);
	}
	return 0;
}

/* Convert apage to bitmap */

static int CONVERT_TO_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start, __d_off end)
{
	unsigned i;
	COPY_APAGE(fs, fs->tmp_map, map);
	MAKE_APAGE_BITMAP(fs, map, start);
	for (i = sizeof(struct aentry); i < fs->apage_size; i += sizeof(struct aentry)) {
		struct aentry *ae = GET_AENTRY_VALID(fs, fs->tmp_map, i);
		__d_off st;
		unsigned len = ae->len;
		unsigned Xoff, Xlen;
		unsigned shift;
		if (__unlikely(!len))
			continue;
		st = ae->start;
		if (__unlikely(st < start) || __unlikely(st + len < st) || __unlikely(st + len > end)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "POINTER (%LX,%X) OUT OF RANGE (%LX-%LX)", (unsigned long long)st, len, (unsigned long long)start, (unsigned long long)end);
			return -EFSERROR;
		}
		shift = fs->sectors_per_block_bits;
		if (__unlikely(((unsigned)st | (unsigned)len) & ((1 << shift) - 1))) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "POINTER (%LX,%X) NOT ALIGNED", (unsigned long long)st, len);
		}
		Xoff = (st - start) >> shift;
		Xlen = len >> shift;
		do {
			if (__unlikely(!BMP_TEST(fs, map, Xoff))) {
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BLOCK %LX+%X FREED TWICE", (unsigned long long)st, (Xoff << shift) - (unsigned)(st - start));
				return -EFSERROR;
			}
			BMP_CLEAR(fs, map, Xoff);
			Xoff++;
		} while (--Xlen);
	}
	return 0;
}

