#include "spadfs.h"

#define DECREASE_FREESPACE	0
#define INCREASE_FREESPACE	1

static __finline__ int GROUP_ACTION_INLINE(SPADFS *fs, unsigned group, unsigned len, int action)
{
	if (__likely(action == INCREASE_FREESPACE)) {
		if (__unlikely(fs->group_info[group].freespace + len > (__d_off)1 << fs->sectors_per_group_bits)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOCATION COUNT FOR GROUP %u OVERFLOW: %LX, %X", group, (unsigned long long)fs->group_info[group].freespace, len);
			return -EFSERROR;
		}
		fs->group_info[group].freespace += len;
		fs->group_info[group].zone->freespace += len;
	} else {
		if (__unlikely(fs->group_info[group].freespace < len) || __unlikely(fs->group_info[group].zone->freespace < len)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOCATION COUNT FOR GROUP %u UNDERFLOW: %LX, %LX, %X", group, (unsigned long long)fs->group_info[group].freespace, (unsigned long long)fs->group_info[group].zone->freespace, len);
			return -EFSERROR;
		}
		fs->group_info[group].freespace -= len;
		fs->group_info[group].zone->freespace -= len;
	}
	return 0;
}

static int GROUP_ACTION_NOINLINE(SPADFS *fs, unsigned group, unsigned len, int action)
{
	return GROUP_ACTION_INLINE(fs, group, len, action);
}

static int GROUP_ACTION(SPADFS *fs, __d_off start, unsigned len, int action)
{
	unsigned start_group, end_group;
	start_group = start >> fs->sectors_per_group_bits;
	end_group = (start + (len - 1)) >> fs->sectors_per_group_bits;
	if (__unlikely(end_group >= fs->n_groups)) {
		range_error:
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "GROUP_ACTION: INVALID RANGE (%LX,%X)", (unsigned long long)start, len);
		return -EFSERROR;
	}
	if (__unlikely(start_group > end_group)) {
		goto range_error;
	}
	if (__likely(start_group == end_group)) {
		return GROUP_ACTION_INLINE(fs, start_group, len, action);
	} else {
		int r = GROUP_ACTION_NOINLINE(fs, start_group, fs->group_mask + 1 - (start & fs->group_mask), action);
		if (__unlikely(r)) return r;
		for (start_group++; start_group < end_group; start_group++) {
			r = GROUP_ACTION_NOINLINE(fs, start_group, fs->group_mask + 1, action);
			if (__unlikely(r)) return r;
		}
		r = GROUP_ACTION_NOINLINE(fs, start_group, ((start + len - 1) & fs->group_mask) + 1, action);
		return r;
	}
}

static int MAP_2_APAGE_N(SPADFS *fs, APAGE_MAP *map)
{
	unsigned i, j;
	if (map == fs->tmp_map) return -1;
	for (i = 0; i < fs->n_apages; i++) for (j = 0; j < 2; j++) if (fs->apage_info[i].map[j] == map) return i;
	panic("MAP_2_APAGE_N: INVALID POINTER %p", map);
}

static inline struct aentry *GET_AENTRY(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	if (__unlikely(off & (sizeof(struct aentry) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "UNALIGNED APAGE ENTRY %u, APAGE %d", off, MAP_2_APAGE_N(fs, map));
		return ERR_PTR(-EFSERROR);
	}
	if (__unlikely(off >= fs->apage_size)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE ENTRY %u TOO BIG, APAGE %d", off, MAP_2_APAGE_N(fs, map));
		return ERR_PTR(-EFSERROR);
	}
	return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_block_bits)].entry + (off & (fs->blocksize - 1)));
}

static inline struct aentry *GET_AENTRY_VALID(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	if (__unlikely(off & (sizeof(struct aentry) - 1))) {
		BUG();
	}
	if (__unlikely(off >= fs->apage_size)) {
		BUG();
	}
	return (struct aentry *)((char *)map[off >> (9 + fs->sectors_per_block_bits)].entry + (off & (fs->blocksize - 1)));
}

static inline struct apage_head *GET_HEAD(APAGE_MAP *map)
{
	return (struct apage_head *)map[0].entry;
}

static void FREE_APAGE_BUFFERS(APAGE_MAP *map, unsigned n)
{
	while (n--) __brelse(map++->bh);
}

static int READ_APAGE_BUFFERS(SPADFS *fs, APAGE_MAP *map, __d_off sec, unsigned n)
{
	unsigned i;
	for (i = 0; i < n; i++) {
		if (__unlikely(IS_ERR(map[i].entry = SPADFS_READ_SECTOR(fs, sec, &map[i].bh, "READ_APAGE_BUFFERS")))) {
			int r = PTR_ERR(map[i].entry);
			FREE_APAGE_BUFFERS(map, i);
			return r;
		}
		sec += 1U << fs->sectors_per_block_bits;
	}
	return 0;
}

#define invalid_flags(flags, fs)	\
	__unlikely(((flags) & (APAGE_SIZE_BITS | APAGE_BLOCKSIZE_BITS)) != ((((fs)->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | ((fs)->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT)))

#define MAP_READ	0
#define MAP_NEW		1
#define MAP_WRITE	2
#define MAP_ALLOC	3

static APAGE_MAP *MAP_APAGE(SPADFS *fs, unsigned ap, int flags, APAGE_MAP **other)
{
	struct apage_info *info;
	unsigned idx, i;
	int r;
	__d_off sec;
	struct apage_head *a;
	struct apage_head *aa;
	sec = fs->apage_index[ap].apage;
	if (__unlikely((unsigned)sec & ((1U << fs->sectors_per_page_bits) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "MISALIGNED APAGE %u AT BLOCK %LX", ap, (unsigned long long)sec);
		return ERR_PTR(-EFSERROR);
	}
	info = &fs->apage_info[ap];
	if (__unlikely(r = READ_APAGE_BUFFERS(fs, info->map[0], sec, fs->n_apage_mappings))) return ERR_PTR(r);
	if (__unlikely(r = READ_APAGE_BUFFERS(fs, info->map[1], sec + (1 << (fs->sectors_per_page_bits - 1)), fs->n_apage_mappings))) {
		FREE_APAGE_BUFFERS(info->map[0], fs->n_apage_mappings);
		return ERR_PTR(r);
	}

	a = GET_HEAD(info->map[0]);
	if (__unlikely(a->magic != APAGE_MAGIC)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BAD MAGIC ON APAGE %LX", (unsigned long long)sec);
		r = -EFSERROR;
		goto ret_err;
	}

	idx = !CC_VALID(fs, a->cc, a->txc);

	if (__unlikely(flags == MAP_NEW)) goto read_ok;

	if (invalid_flags(GET_HEAD(info->map[idx])->s.u.l.flags, fs)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE %LX HAS INVALID FLAGS %02X", (unsigned long long)sec, GET_HEAD(info->map[idx])->s.u.l.flags);
		r = -EFSERROR;
		goto ret_err;
	}

	if (__unlikely(flags == MAP_READ)) {
		read_ok:
		if (__likely(other != NULL)) *other = info->map[idx ^ 1];
		else FREE_APAGE_BUFFERS(info->map[idx ^ 1], fs->n_apage_mappings);
		return info->map[idx];
	}
	if (__likely(flags == MAP_ALLOC)) {
		aa = GET_HEAD(info->map[idx]);
		if (__unlikely(!(aa->s.u.l.flags & APAGE_BITMAP)) && __unlikely(!aa->s.u.l.last)) {
			r = -ENOSPC;
			goto ret_err;
		}
	}
	if (__likely(CC_CURRENT(fs, a->cc, a->txc))) {
		if (invalid_flags(GET_HEAD(info->map[idx ^ 1])->s.u.l.flags, fs)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "APAGE %LX HAS INVALID FLAGS %02X IN INACTIVE PART", (unsigned long long)sec, GET_HEAD(info->map[idx ^ 1])->s.u.l.flags);
			r = -EFSERROR;
			goto ret_err;
		}
		goto read_ok;
	}
	start_atomic_buffer_modify(info->map[0][0].bh);
	CC_SET_CURRENT(fs, &a->cc, &a->txc);
	end_atomic_buffer_modify(info->map[0][0].bh);

	for (i = 0; i < fs->n_apage_mappings; i++) {
		char *to, *from;
		unsigned size;
		to = (char *)info->map[idx ^ 1][i].entry;
		from = (char *)info->map[idx][i].entry;
		size = fs->blocksize;
		if (!i) {
			if (size > fs->apage_size) size = fs->apage_size;
			to += sizeof(struct apage_head) - sizeof(struct apage_subhead);
			from += sizeof(struct apage_head) - sizeof(struct apage_subhead);
			size -= sizeof(struct apage_head) - sizeof(struct apage_subhead);
		}
		memcpy(to, from, size);
		mark_buffer_dirty(info->map[idx ^ 1][i].bh);
	}
	idx ^= 1;
	goto read_ok;

	ret_err:
	FREE_APAGE_BUFFERS(info->map[0], fs->n_apage_mappings);
	FREE_APAGE_BUFFERS(info->map[1], fs->n_apage_mappings);
	return ERR_PTR(r);
}

static void UNMAP_APAGE(SPADFS *fs, APAGE_MAP *map, int modified)
{
	BUG_ON(!atomic_read(&map[0].bh->b_count));
	if (modified) {
		unsigned i;
		GET_HEAD(map)->s.u.l.flags &= ~APAGE_CHECKSUM_VALID;
		for (i = 0; i < fs->n_apage_mappings; i++) mark_buffer_dirty(map[i].bh);
	}
	FREE_APAGE_BUFFERS(map, fs->n_apage_mappings);
}

static void COPY_APAGE(SPADFS *fs, APAGE_MAP *dst, APAGE_MAP *src)
{
	unsigned i;
	unsigned n = fs->n_apage_mappings;
	unsigned size = fs->blocksize;
	if (__unlikely(fs->apage_size < size)) size = fs->apage_size;
	for (i = 0; i < n; i++) memcpy(dst[i].entry, src[i].entry, size);
}

static void MAKE_APAGE(SPADFS *fs, APAGE_MAP *map)
{
	unsigned i;
	struct apage_head *head;
	struct aentry *ae;
	head = GET_HEAD(map);
	head->s.u.l.flags = ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | (fs->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT);
	head->s.u.l.checksum = 0;
	head->s.u.l.freelist = sizeof(struct aentry);
	head->s.u.l.last = 0;
	head->s.u.l.first = 0;
	i = sizeof(struct aentry);
	do {
		ae = GET_AENTRY_VALID(fs, map, i);
		ae->start = 0;
		ae->len = 0;
		ae->prev = 0;
		ae->next = i += sizeof(struct aentry);
	} while (i < fs->apage_size);
	ae->next = 0;
}

static void MAKE_APAGE_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start)
{
	unsigned i;
	struct apage_head *head;
	struct aentry *ae;
	head = GET_HEAD(map);
	head->s.u.b.flags = ((fs->sectors_per_page_bits - 1) << APAGE_SIZE_BITS_SHIFT) | (fs->sectors_per_block_bits << APAGE_BLOCKSIZE_BITS_SHIFT) | APAGE_BITMAP;
	head->s.u.b.checksum = 0;
	head->s.u.b.start1 = MAKE_PART_1(start);
	head->s.u.b.start0 = MAKE_PART_0(start);
	i = sizeof(struct aentry);
	do {
		ae = GET_AENTRY_VALID(fs, map, i);
		memset(ae, 0xff, sizeof(struct aentry));
		i += sizeof(struct aentry);
	} while (i < fs->apage_size);
}

static int GET_MAP_STATS(SPADFS *fs, APAGE_MAP *map, __d_off *freespace)
{
	struct apage_head *head = GET_HEAD(map);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		unsigned p = head->s.u.l.first;
		unsigned max_loop = (512 / sizeof(struct aentry) / 2) << fs->sectors_per_page_bits;
		while (p) {
			struct aentry *aentry = GET_AENTRY(fs, map, p);
			if (__unlikely(IS_ERR(aentry))) return PTR_ERR(aentry);
			if (__unlikely(GROUP_ACTION(fs, aentry->start, aentry->len, INCREASE_FREESPACE))) return -EFSERROR;
			*freespace += aentry->len;
			p = aentry->next;
			if (__unlikely(!--max_loop)) {
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "GET_MAP_STATS: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
				return -EFSERROR;
			}
		}
		return 0;
	} else {
		unsigned p;
		__d_off bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1);
		for (p = sizeof(struct aentry); p < fs->apage_size; p += sizeof(struct aentry)) {
			unsigned i;
			__u8 *bm = (char *)GET_AENTRY(fs, map, p);
			if (__unlikely(IS_ERR(bm))) return PTR_ERR(bm);
			for (i = 0; i < sizeof(struct aentry) * 8; i++) {
				if (BITMAP_TEST_32_FULL(bm - sizeof(struct apage_head), i)) {
					i += 31;
					continue;
				}
				if (!BITMAP_TEST(bm - sizeof(struct apage_head), i)) {
					*freespace += fs->blocksize / 512;
					if (__unlikely(GROUP_ACTION(fs, bst + ((i + (p - sizeof(struct aentry)) * 8) << fs->sectors_per_block_bits), fs->blocksize / 512, INCREASE_FREESPACE))) return -EFSERROR;
				}
			}
		}
		return 0;
	}
}

int SPADFS_COUNT_FREE_SPACE(SPADFS *fs)
{
	unsigned i;
	__d_off freespace = 0;
	for (i = 0; i < fs->n_active_apages; i++) {
		int r;
		APAGE_MAP *map;
		if (__unlikely(IS_ERR(map = MAP_APAGE(fs, i, MAP_READ, NULL)))) {
			return PTR_ERR(map);
		}
		if (__unlikely(r = GET_MAP_STATS(fs, map, &freespace))) return r;
		UNMAP_APAGE(fs, map, 0);
	}
	fs->freespace = freespace;

	freespace = 0;
	for (i = 0; i < fs->n_groups; i++) {
		freespace += fs->group_info[i].freespace;
	}
	if (__unlikely(freespace != fs->freespace)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE SPACE IN GROUPS MISCOUNTED, %LX != %LX", (unsigned long long)freespace, (unsigned long long)fs->freespace);
		return -EFSERROR;
	}

	freespace = 0;
	for (i = 0; i < 3; i++) {
		freespace += fs->zones[i].freespace;
	}
	if (__unlikely(freespace != fs->freespace)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE SPACE IN ZONES MISCOUNTED, %LX != %LX", (unsigned long long)freespace, (unsigned long long)fs->freespace);
		return -EFSERROR;
	}


	return 0;
}

static int FIND_BLOCK_BEFORE(SPADFS *fs, APAGE_MAP *map, __d_off from)
{
	unsigned i;
	unsigned rlimit;
	__d_off xst = 0;
	unsigned st = 0, nx;
	struct aentry *sta = (struct aentry *)GET_HEAD(map), *nxa;
	for (i = 4 * sizeof(struct aentry); i <= fs->apage_size - 4 * sizeof(struct aentry); i += 124 * sizeof(struct aentry)) {
		unsigned j;
		for (j = 0; j < 4; j++) {
			struct aentry *a = GET_AENTRY_VALID(fs, map, i);
			if (__unlikely(IS_ERR(a))) return PTR_ERR(a);
			if (a->len && a->start <= from && __unlikely(a->start >= xst)) xst = a->start, st = i, sta = a;
			i += sizeof(struct aentry);
		}
	}
	rlimit = fs->apage_size;
	do {
		nx = sta->next;
		nxa = GET_AENTRY(fs, map, nx);
		if (__unlikely(IS_ERR(nxa))) return PTR_ERR(nxa);
		if (__unlikely(!nx) || nxa->start > from) return st;
		st = nx;
		sta = nxa;
	} while (__likely(rlimit -= sizeof(struct aentry)));
	SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FIND_BLOCK_BEFORE: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
	return -EFSERROR;
}

static inline int BMP_TEST(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return (p[(off % (sizeof(struct aentry) * 8)) >> 5] >> (off & 31)) & 1;
}

static inline int BMP_SET(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] |= 1 << (off & 31);
}

static inline int BMP_CLEAR(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] &= ~(1 << (off & 31));
}

static inline int BMP_TEST_32_FULL(SPADFS *fs, APAGE_MAP *map, unsigned off)
{
	__u32 *p = (__u32 *)GET_AENTRY_VALID(fs, map, (1 + off / (sizeof(struct aentry) * 8)) * sizeof(struct aentry));
	return p[(off % (sizeof(struct aentry) * 8)) >> 5] == 0xffffffffu;
}

static int CHECK_OTHER_MAP_BMP(SPADFS *fs, APAGE_MAP *map, __d_off off, unsigned n_sec, __d_off *next_free)
{
	struct apage_head *head = GET_HEAD(map);
	unsigned Xboff = BITMAP_OFFSET(head, off);
	unsigned Xn_sec = BITMAP_LEN(head, n_sec);
	if (__unlikely(Xboff + Xn_sec <= Xboff) || __unlikely(Xboff + Xn_sec > BITMAP_SIZE(fs->apage_size))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "CHECK_OTHER_MAP_BMP: BITMAP ERROR, APAGE %d, XBOFF %X, OFF %LX, N_SEC %X", MAP_2_APAGE_N(fs, map), Xboff, (unsigned long long)off, n_sec);
		return -EFSERROR;
	}
	do {
		if (__unlikely(BMP_TEST(fs, map, Xboff))) {
			while (++Xboff < BITMAP_SIZE(fs->apage_size))
				if (!BMP_TEST(fs, map, Xboff)) {
					*next_free = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1) + (Xboff << fs->sectors_per_block_bits);
					return -EBUSY;
				}
			return -EBUSY;
		}
		Xboff++;
	} while (--Xn_sec);
	return 0;
}

static int CHECK_OTHER_MAP(SPADFS *fs, APAGE_MAP *map, __d_off off, unsigned n_sec, __d_off *next_free)
{
	if (__unlikely(!map)) return 0;
	*next_free = 0;
	if (__likely(!(GET_HEAD(map)->s.u.l.flags & APAGE_BITMAP))) {
		int preblk, postblk;
		struct aentry *e;
		preblk = FIND_BLOCK_BEFORE(fs, map, off);
		if (__unlikely(preblk < 0)) return preblk;
		e = GET_AENTRY_VALID(fs, map, preblk);
		if (__likely(preblk != 0)) {
			if (off + n_sec <= e->start + e->len) {
				return 0;
			}
		}
		postblk = e->next;
		if (__likely(postblk != 0)) {
			e = GET_AENTRY(fs, map, postblk);
			if (__unlikely(IS_ERR(e))) return PTR_ERR(e);
			*next_free = e->start;
		}
		return -EBUSY;
	} else {
		return CHECK_OTHER_MAP_BMP(fs, map, off, n_sec, next_free);
	}
}

static int ALLOC_BLOCKS_IN_APAGE_BMP(SPADFS *fs, APAGE_MAP *map, APAGE_MAP *other, struct alloc *al)
{
	struct apage_head *head = GET_HEAD(map);
	unsigned shift = fs->sectors_per_block_bits;
	unsigned Xbmax = BITMAP_SIZE(fs->apage_size);
	unsigned Xboff;
	__d_off bst = MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1);
	__d_off bottom, top;
	bottom = al->sector;
	if (bottom < bst) bottom = bst;
	top = al->top;
	if (__likely(top > bst + (Xbmax << shift))) top = bst + (Xbmax << shift);
	if (__unlikely(bottom >= top)) return -ENOSPC;
	Xboff = (bottom - bst) >> shift;
	if (top - bst < (Xbmax << shift)) Xbmax = (top - bst) >> shift;
	while (Xboff < Xbmax) {
		if (__likely(BMP_TEST_32_FULL(fs, map, Xboff))) {
			Xboff = (Xboff + 32) & ~31;
			continue;
		}
		test_bit:
		if (__unlikely(!BMP_TEST(fs, map, Xboff))) {
			__d_off new_bot;
			unsigned Xc;
			if (__unlikely(al->flags & ALLOC_METADATA)) {
				if ((((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags)) + al->n_sectors > ALLOC_MASK(al->flags) + 1) goto align;
			} else if (((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags)) {
				unsigned Xplus;
				align:
				Xplus = (ALLOC_MASK(al->flags) + 1 - (((unsigned)bst + (Xboff << shift)) & ALLOC_MASK(al->flags))) >> shift;
				if (__unlikely(!Xplus)) Xplus = 1;
				Xboff += Xplus;
				continue;
			}
			for (Xc = 1; Xc < al->n_sectors >> shift; Xc++)
				if (__unlikely(Xboff + Xc >= Xbmax) || BMP_TEST(fs, map, Xboff + Xc)) {
					Xboff += Xc + 1;
					goto cont;
				}
			if (__unlikely(CHECK_OTHER_MAP(fs, other, bst + (Xboff << shift), al->n_sectors, &new_bot))) {
				if (__likely(new_bot > bst + (Xboff << shift)) && __likely(new_bot < bst + (Xbmax << shift))) {
					Xboff = (new_bot - bst + ((1 << shift) - 1)) >> shift;
					continue;
				}
				return -ENOSPC;
			}
			al->sector = bst + (Xboff << shift);
			for (Xc = 0; Xc < al->n_sectors >> shift; Xc++) BMP_SET(fs, map, Xboff + Xc);
			return 0;
		}
		Xboff++;
		if (__likely(Xboff < Xbmax) && __likely(Xboff & 31)) goto test_bit;
		cont:;
	}
	return -ENOSPC;
}

static inline void DELETE_BLOCK(SPADFS *fs, APAGE_MAP *map, struct aentry *e)
{
	struct aentry *pre, *post;
	struct apage_head *head;
	unsigned nblk;
	pre = GET_AENTRY(fs, map, e->prev);
	if (__unlikely(IS_ERR(pre))) return;
	pre->next = e->next;
	post = GET_AENTRY(fs, map, e->next);
	if (__unlikely(IS_ERR(post))) return;
	nblk = post->prev;
	post->prev = e->prev;
	e->start = 0;
	e->len = 0;
	e->prev = 0;
	head = GET_HEAD(map);
	e->next = head->s.u.l.freelist;
	head->s.u.l.freelist = nblk;
}

static inline int ALLOC_BLOCK(SPADFS *fs, struct aentry *a, struct alloc *al, APAGE_MAP *other)
{
	__d_off bottom, top, new_bot;
	bottom = al->sector;
	if (__likely(bottom < a->start)) bottom = a->start;
	top = al->top;
	if (__likely(top > a->start + a->len)) top = a->start + a->len;
	new_bottom:
	if (__unlikely(al->flags & ALLOC_METADATA))
		if (__likely(((unsigned)bottom & ALLOC_MASK(al->flags)) + al->n_sectors <= ALLOC_MASK(al->flags) + 1)) goto skip_pad;
	bottom = (bottom + ALLOC_MASK(al->flags)) & ~(__d_off)ALLOC_MASK(al->flags);
	skip_pad:
	if (__unlikely(bottom + al->n_sectors > top)) return -ENOSPC;

	if (__unlikely(CHECK_OTHER_MAP(fs, other, bottom, al->n_sectors, &new_bot))) {
		if (__likely(new_bot > bottom)) {
			bottom = new_bot;
			goto new_bottom;
		}
		return -ENOSPC;
	}
	if (__unlikely(bottom > a->start)) {
		al->flags |= ALLOC_FREE_FROM;
		al->top = a->start;
	}
	a->len -= ((__u32)bottom - (__u32)a->start) + al->n_sectors;
	a->start = bottom + al->n_sectors;
	al->sector = bottom;
	return 0;
}

static int ALLOC_BLOCKS_IN_APAGE(SPADFS *fs, APAGE_MAP *map, APAGE_MAP *other, struct alloc *al)
{
	struct apage_head *head = GET_HEAD(map);
	if (__unlikely(IS_ERR(head))) return PTR_ERR(head);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		struct aentry *e;
		int n_cycles;
		int blk = FIND_BLOCK_BEFORE(fs, map, al->sector);
		if (__unlikely(blk < 0)) return blk;
		e = GET_AENTRY_VALID(fs, map, blk);
		n_cycles = fs->apage_size;
		next_try:
		if (!blk || al->sector >= e->start + e->len) {
			blk = e->next;	/* valid from FIND_BLOCK_BEFORE */
			if (__unlikely(!blk)) return -ENOSPC;
			e = GET_AENTRY_VALID(fs, map, blk);
		}
		if (__unlikely(e->start >= al->top)) return -ENOSPC;
		if (e->len >= al->n_sectors && __likely(!ALLOC_BLOCK(fs, e, al, other))) {
			if (__unlikely(!e->len)) DELETE_BLOCK(fs, map, e);
			return 0;
		}
		blk = e->next;
		if (__unlikely(!blk)) return -ENOSPC;
		e = GET_AENTRY(fs, map, blk);
		if (__unlikely(blk < 0)) return blk;
		if (__likely(n_cycles -= sizeof(struct aentry))) goto next_try;
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALLOC_BLOCKS_IN_APAGE: INFINITE LOOP IN APAGE %d", MAP_2_APAGE_N(fs, map));
		return -EFSERROR;
	} else {
		return ALLOC_BLOCKS_IN_APAGE_BMP(fs, map, other, al);
	}
}

static __finline__ int TRY_APAGE(SPADFS *fs, unsigned ap, struct alloc *al)
{
	APAGE_MAP *map, *other;
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_ALLOC, &other)))) return PTR_ERR(map);
	if (__likely(!ALLOC_BLOCKS_IN_APAGE(fs, map, other, al))) {
		UNMAP_APAGE(fs, map, 1);
		UNMAP_APAGE(fs, other, 0);
		return 0;
	}
	UNMAP_APAGE(fs, map, 0);
	UNMAP_APAGE(fs, other, 0);
	return -ENOSPC;
}

static int ADDR_2_APAGE(SPADFS *fs, __d_off o, char *msg)
{
	int a1, a2, a;
	a2 = fs->n_active_apages - 1;
	a1 = 0;
	again:
	a = (a1 + a2) >> 1;
	if (o < (__d_off)fs->apage_index[a - 1].end_block) a2 = a - 1;
	else if (o >= (__d_off)fs->apage_index[a].end_block) a1 = a + 1;
	else return a;
	if (__likely(a1 <= a2)) goto again;
	SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "CAN'T FIND APAGE FOR BLOCK %LX, STUCK ON %d/%d, CALLED FROM %s", (unsigned long long)o, a1, a2, msg);
	return -EFSERROR;
}

static int ALLOC_BLOCKS_IN_RANGE(SPADFS *fs, struct alloc *al)
{
	int ap, tap;
	if (__unlikely(al->top > fs->size)) al->top = fs->size;
	if (__unlikely(al->sector >= al->top)) return -ENOSPC;
	ap = ADDR_2_APAGE(fs, al->sector, "ALLOC_BLOCKS_IN_RANGE 1");
	if (__unlikely(ap < 0)) return ap;
	if (__likely(al->top <= fs->apage_index[ap].end_block)) tap = ap;
	else {
		tap = ADDR_2_APAGE(fs, al->top - 1, "ALLOC_BLOCKS_IN_RANGE 2");
		if (__unlikely(tap < 0)) return tap;
	}
	next_ap:
	if (__likely(!TRY_APAGE(fs, ap, al))) return 0;
	ap++;
	if (__unlikely(ap <= tap)) goto next_ap;
	return -ENOSPC;
}

static int ALLOC_BLOCKS_IN_DIFFERENT_GROUP(SPADFNODE *f, struct alloc *al);

static unsigned GET_BLOCKLEN_AT(SPADFS *fs, __d_off block, unsigned max)
{
	struct apage_head *head;
	APAGE_MAP *map;
	int ret;
	int ap = ADDR_2_APAGE(fs, block, "GET_BLOCKLEN_AT");
	if (__unlikely(ap < 0)) return 0;
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_READ, NULL)))) return 0;
	head = GET_HEAD(map);
	if (__unlikely(IS_ERR(head))) goto unmap_0;
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		struct aentry *e;
		int blk = FIND_BLOCK_BEFORE(fs, map, block);
		if (__unlikely(blk < 0)) goto unmap_0;
		e = GET_AENTRY_VALID(fs, map, blk);
		if (block >= e->start + e->len) goto unmap_0;
		ret = e->start + e->len - block;
		if (__likely(ret > max)) ret = max;
		unmap_ret:
		UNMAP_APAGE(fs, map, 0);
		return ret;
	} else {
		unsigned Xboff = BITMAP_OFFSET(head, block);
		unsigned add = fs->blocksize / 512;
		ret = 0;
		while (ret < max && __likely(Xboff < BITMAP_SIZE(fs->apage_size)) && !BMP_TEST(fs, map, Xboff)) ret += add, Xboff++;
		goto unmap_ret;

	}

	unmap_0:
	UNMAP_APAGE(fs, map, 0);
	return 0;
}
	

int SPADFS_ALLOC_BLOCKS(SPADFNODE *f, struct alloc *al)
{
	SPADFS *fs = f->fs;
	down(&fs->alloc_lock);

	if (__unlikely(al->n_sectors & ((1 << fs->sectors_per_block_bits) - 1))) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "TRYING TO ALLOCATE %u BLOCKS", al->n_sectors);
		up(&fs->alloc_lock);
		return -EFSERROR;
	}
	al->sector &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);

	if (__unlikely(al->sector >= fs->size)) al->sector = 0;

	if (__unlikely(al->flags & ALLOC_METADATA)) {
		al->flags |= ((1 << fs->sectors_per_page_bits) - 1) * ALLOC_MASK_1;
	}
	if (al->flags & ALLOC_BIG_FILE) {
		al->flags |= ((1 << fs->sectors_per_cluster_bits) - 1) * ALLOC_MASK_1;
		al->sector &= ~(__d_off)((1 << fs->sectors_per_cluster_bits) - 1);
	}

	if (al->flags & ALLOC_PARTIAL_AT_GOAL) {
		__u32 bl = GET_BLOCKLEN_AT(fs, al->sector, al->n_sectors);
		if (__likely(bl != 0)) {
			unsigned orig_n_sectors = al->n_sectors;
			al->n_sectors = bl;
			while (1) {
				al->n_sectors &= ~(fs->blocksize / 512 - 1);
				if (__unlikely(!al->n_sectors)) break;
				al->top = al->sector + al->n_sectors;
				if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) goto ok_allocated;
				al->n_sectors >>= 1;
			}
			al->n_sectors = orig_n_sectors;
		}
	}

	al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask;

	if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
		ok_allocated:
		if (__unlikely(al->flags & ALLOC_FREE_FROM)) {
			int r;
			if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, al->top, al->sector - al->top, 0))) {
				up(&fs->alloc_lock);
				return r;
			}
		}
		if (__unlikely(al->n_sectors > fs->freespace)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE BLOCK COUNT UNDERRUN: %X < %X", (unsigned)fs->freespace, al->n_sectors);
			up(&fs->alloc_lock);
			return 0;
		}
		fs->freespace -= al->n_sectors;
		GROUP_ACTION(fs, al->sector, al->n_sectors, DECREASE_FREESPACE);
		up(&fs->alloc_lock);
		return 0;
	}
	if (__likely(!ALLOC_BLOCKS_IN_DIFFERENT_GROUP(f, al))) goto ok_allocated;
	up(&fs->alloc_lock);
	return -ENOSPC;
}

static __finline__ __d_off ZONE_AVERAGE_FREE(struct spadfszone *z)
{
	unsigned n = 0;
	do {
		n++;
	} while (1 << n < z->grp_n);
	return z->freespace >> n;
}

static int ALLOC_BLOCKS_IN_DIFFERENT_GROUP(SPADFNODE *f, struct alloc *al)
{
	SPADFS *fs = f->fs;
	__d_off orig_start = al->sector;
	struct spadfszone *z;
	retry_less:
	if (__likely(al->flags & ALLOC_BIG_FILE)) {
		z = &fs->zones[2];
	} else if (__likely(al->flags & ALLOC_SMALL_FILE)) {
		z = &fs->zones[1];
	} else {
		z = &fs->zones[0];
	}
	forward_quad_search:
	if (__likely(z->grp_n)) {
		unsigned pass;
		__d_off avg_free = ZONE_AVERAGE_FREE(z);
		if (__unlikely(!avg_free)) avg_free = al->n_sectors;
		retry:
		for (pass = 0; pass < 2; pass++) {
			unsigned i;
			unsigned grp_current = orig_start >> fs->sectors_per_group_bits;
			if (__unlikely(grp_current < z->grp_start) || __unlikely(grp_current >= z->grp_start + z->grp_n)) grp_current = z->grp_start;
			i = 1;
			do {
				grp_current += __likely(!pass) ? i : 1;
				if (__unlikely(grp_current >= z->grp_start + z->grp_n)) grp_current -= z->grp_n;
				if (fs->group_info[grp_current].freespace >= avg_free) {
					al->sector = (__d_off)grp_current << fs->sectors_per_group_bits;
					al->top = (__d_off)(grp_current + 1) << fs->sectors_per_group_bits;
					if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
						alloc_success_set_group:
						al->flags |= ALLOC_NEW_GROUP_HINT;
						return 0;
					}
				}
				if (__likely(!pass)) i *= 2;
				else i++;
			} while (i <= z->grp_n);
		}
		if (avg_free > al->n_sectors) {
			avg_free = al->n_sectors;
			goto retry;
		}
	}
	try_another_zone:
	if (__likely(al->flags & ALLOC_BIG_FILE)) {
		if (__likely(z == &fs->zones[2])) z = &fs->zones[1];
		else if (__likely(z == &fs->zones[1])) z = &fs->zones[0];
		else goto failed;
	} else if (__likely(al->flags & ALLOC_SMALL_FILE)) {
		if (__likely(z == &fs->zones[1])) z = &fs->zones[2];
		else if (__likely(z == &fs->zones[2])) z = &fs->zones[0];
		else goto failed;
	} else {
		if (__likely(z == &fs->zones[0])) z = &fs->zones[1];
		else if (__likely(z == &fs->zones[1])) z = &fs->zones[2];
		else return -ENOSPC;
	}
	if (z == &fs->zones[2]) goto forward_quad_search;
	if (__likely(z->grp_n)) {
		unsigned grp_current;
		__d_off avg_free = ZONE_AVERAGE_FREE(z) / SPADFS_AVG_FREE_DIVIDE_OTHERZONE;
		if (__unlikely(!avg_free)) avg_free = al->n_sectors;
		retry2:
		grp_current = z->grp_start + z->grp_n;
		do {
			grp_current--;
			if (fs->group_info[grp_current].freespace >= avg_free) {
				al->sector = (__d_off)grp_current << fs->sectors_per_group_bits;
				al->top = (__d_off)(grp_current + 1) << fs->sectors_per_group_bits;
				if (__likely(!ALLOC_BLOCKS_IN_RANGE(fs, al))) {
					goto alloc_success_set_group;
				}
			}
		} while (grp_current > z->grp_start);
		if (avg_free > al->n_sectors) {
			avg_free = al->n_sectors;
			goto retry2;
		}
	}
	goto try_another_zone;
	failed:
	/*
	 * Try to allocate less sectors
	 */
	al->n_sectors = (al->n_sectors >> 1) & ~(fs->blocksize / 512 - 1);
	if (__unlikely(!al->n_sectors)) return -ENOSPC;
	if (__likely(ALLOC_MASK(al->flags))) {
		if (__unlikely(al->n_sectors <= ALLOC_MASK(al->flags))) al->flags = (al->flags & ~(ALLOC_MASK_MASK | ALLOC_BIG_FILE)) | ALLOC_SMALL_FILE;
		else al->n_sectors &= ~ALLOC_MASK(al->flags);
	}
	
	al->sector = orig_start;
	al->top = (al->sector + fs->group_mask + 1) & ~fs->group_mask;
	if (!ALLOC_BLOCKS_IN_RANGE(fs, al)) return 0;
	goto retry_less;
}

static int APAGE_FREE(SPADFS *fs, APAGE_MAP *map, __d_off off, __u32 len)
{
	struct apage_head *head = GET_HEAD(map);
	if (__likely(!(head->s.u.l.flags & APAGE_BITMAP))) {
		int preblk, postblk, newblk;
		struct aentry *pre, *post, *new;
		preblk = FIND_BLOCK_BEFORE(fs, map, off);
		if (__unlikely(preblk < 0)) return preblk;
		pre = GET_AENTRY_VALID(fs, map, preblk);
		postblk = pre->next;	/* already valid */
		post = GET_AENTRY_VALID(fs, map, postblk);
		if (__likely(preblk != 0)) {
			if ((__d_off)pre->start + pre->len == off) {
				if (__unlikely((__u32)(pre->len + len) < (__u32)len)) goto nj1;
				if (__likely(postblk != 0)) {
					if (off + len == (__d_off)post->start) {
						if (__unlikely((__u32)(pre->len + len + post->len) < (__u32)post->len)) goto nj2;
						pre->len += len + post->len;
						DELETE_BLOCK(fs, map, post);
						goto done;
					} else if (__unlikely(off + len > (__d_off)post->start)) {
						goto post_over;
					}
				}
				nj2:
				pre->len += len;
				goto done;
			} else if (__unlikely((__d_off)pre->start + pre->len > off)) {
				postblk = preblk;
				post = pre;
				goto post_over;
			}
		}
		nj1:
		if (__likely(postblk != 0)) {
			if (off + len == (__d_off)post->start) {
				if (__unlikely((__u32)(post->len + len) < (__u32)len)) goto nj3;
				post->start = off;
				post->len += len;
				goto done;
			} else if (__unlikely(off + len > (__d_off)post->start)) {
				post_over:
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "FREE (%LX,%X) DOES OVERLAP WITH BLOCK (%LX,%X)", (unsigned long long)off, (unsigned)len, (unsigned long long)post->start, (unsigned)post->len);
				return -EFSERROR;
			}
		}
		nj3:
		if (__unlikely(!(newblk = head->s.u.l.freelist))) return 1;
		new = GET_AENTRY(fs, map, newblk);
		if (__unlikely(IS_ERR(new))) return PTR_ERR(new);
		head->s.u.l.freelist = new->next;
		new->start = off;
		new->len = len;
		new->prev = preblk;
		new->next = postblk;
		pre->next = newblk;
		post->prev = newblk;

		done:
		return 0;
	} else {
		unsigned bmpoff = BITMAP_OFFSET(head, off);
		len = BITMAP_LEN(head, len);
		if (__unlikely(bmpoff + len <= bmpoff) || __unlikely(bmpoff + len > BITMAP_SIZE(fs->apage_size))) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BAD BITMAP OFFSET: %LX,%X FREE(%LX,%X)", (unsigned long long)MAKE_D_OFF(head->s.u.b.start0, head->s.u.b.start1), bmpoff, (unsigned long long)off, (unsigned)len);
			return -EFSERROR;
		}
		do {
			BMP_CLEAR(fs, map, bmpoff);
			bmpoff++;
		} while (--len);
		return 0;
	}
}

static int SPLIT_APAGE(SPADFS *fs, APAGE_MAP *map, int ap);
static int CONVERT_TO_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start, __d_off end);

int SPADFS_FREE_BLOCKS_(SPADFNODE *f, __d_off start, __d_off n_blocks, int acct)
{
	int ap;
	int r;
	SPADFS *fs = f->fs;
	APAGE_MAP *map;
	if (__unlikely(!n_blocks) || __unlikely(((unsigned)n_blocks & ((1 << fs->sectors_per_block_bits) - 1)) != 0) || __unlikely(start <= SUPERBLOCK_SECTOR) || __unlikely(start >= fs->size) || __unlikely(start + n_blocks > fs->size)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "SPADFS_FREE_BLOCKS: %LX, %LX", (unsigned long long)start, (unsigned long long)n_blocks);
		return -EFSERROR;
	}
	retry:
	if (__unlikely(n_blocks & ~(__d_off)0xffffffffu)) {
		if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, start, 0xffffffffu, acct))) return r;
		start += 0xffffffffu;
		n_blocks -= 0xffffffffu;
		goto retry;
	}
	retry_32:
	ap = ADDR_2_APAGE(fs, start, "SPADFS_FREE_BLOCKS_");
	if (__unlikely(ap < 0)) return ap;
	if ((__d_off)fs->apage_index[ap].end_block < start + n_blocks) {
		__u32 ss = (__d_off)fs->apage_index[ap].end_block - start;
		if (__unlikely(!ss) || __unlikely(ss >= n_blocks)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "SPADFS_FREE_BLOCKS: %LX,%LX - APAGE %d, APAGE_END %LX", (unsigned long long)start, (unsigned long long)n_blocks, ap, (unsigned long long)fs->apage_index[ap].end_block);
			return -EFSERROR;
		}
		if (__unlikely(r = SPADFS_FREE_BLOCKS_(f, start, ss, acct))) return r;
		start += ss;
		n_blocks -= ss;
		goto retry_32;
	}
	if (__unlikely(IS_ERR(map = MAP_APAGE(fs, ap, MAP_WRITE, NULL)))) return PTR_ERR(map);
	free_again:
	r = APAGE_FREE(fs, map, start, n_blocks);
	if (__likely(r <= 0)) {
		UNMAP_APAGE(fs, map, 1);
		if (__likely(!r) && __likely(acct)) {
			fs->freespace += n_blocks;
			GROUP_ACTION(fs, start, n_blocks, INCREASE_FREESPACE);
		}
		return r;
	}
	if (__likely((__d_off)fs->apage_index[ap].end_block - (__d_off)fs->apage_index[ap - 1].end_block > BITMAP_SIZE(fs->apage_size) << fs->sectors_per_block_bits)) {
		if (__unlikely(r = SPLIT_APAGE(fs, map, ap))) return r;
		goto retry_32;
	}
	if (__unlikely(r = CONVERT_TO_BITMAP(fs, map, (__d_off)fs->apage_index[ap - 1].end_block, (__d_off)fs->apage_index[ap].end_block))) {
		UNMAP_APAGE(fs, map, 1);
		return r;
	}
	goto free_again;
}

static int WRITE_APAGE_INDEX(SPADFS *fs);

static int SPLIT_APAGE(SPADFS *fs, APAGE_MAP *map0, int ap)
{
	int r;
	unsigned i, n;
	struct aentry *ae;
	__d_off split_block;
	APAGE_MAP *map1;
	struct apage_index_entry aie;
	unsigned b_s;
	if (__unlikely(fs->n_active_apages >= fs->n_apages)) {
		SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "ALL %d APAGES USED", fs->n_active_apages);
		r = -EFSERROR;
		ret0:
		UNMAP_APAGE(fs, map0, 1);
		return r;
	}
	n = 0;
	ae = (struct aentry *)GET_HEAD(map0);
	for (i = fs->apage_size / (2 * sizeof(struct aentry)); i; i--) {
		n = ae->next;
		ae = GET_AENTRY(fs, map0, n);
		if (__unlikely(IS_ERR(ae))) {
			r = PTR_ERR(ae);
			goto ret0;
		}
	}
	if (__unlikely(!n)) split_block = ((__d_off)fs->apage_index[ap - 1].end_block + (__d_off)fs->apage_index[ap].end_block) >> 1;
	else split_block = ae->start;
	split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	b_s = (BITMAP_SIZE(fs->apage_size) >> 1 << fs->sectors_per_block_bits);
	if (__unlikely(split_block - (__d_off)fs->apage_index[ap - 1].end_block < b_s)) {
		split_block = (__d_off)fs->apage_index[ap - 1].end_block + b_s;
		split_block += (1 << fs->sectors_per_block_bits) - 1;
		split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	}
	if (__unlikely((__d_off)fs->apage_index[ap].end_block - split_block < b_s)) {
		split_block = (__d_off)fs->apage_index[ap].end_block - b_s;
		split_block &= ~(__d_off)((1 << fs->sectors_per_block_bits) - 1);
	}
	COPY_APAGE(fs, fs->tmp_map, map0);
	map1 = MAP_APAGE(fs, fs->n_active_apages, MAP_NEW, NULL);
	if (__unlikely(IS_ERR(map1))) {
		r = PTR_ERR(map1);
		goto ret0;
	}
	MAKE_APAGE(fs, map0);
	MAKE_APAGE(fs, map1);
	for (i = sizeof(struct aentry); i < fs->apage_size; i += sizeof(struct aentry)) {
		__d_off st;
		struct aentry *ae = GET_AENTRY_VALID(fs, fs->tmp_map, i);
		if (__unlikely(!ae->len)) continue;
		st = ae->start;
		if (st >= split_block) r = APAGE_FREE(fs, map1, st, ae->len);
		else if (st + ae->len <= split_block) r = APAGE_FREE(fs, map0, st, ae->len);
		else {
			r = APAGE_FREE(fs, map0, st, split_block - st);
			if (__likely(!r)) r = APAGE_FREE(fs, map1, split_block, st + ae->len - split_block);
		}
		if (__unlikely(r)) {
			if (r > 0) SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "OUT OF SPACE WHEN SPLITTING APAGE"), r = -EFSERROR;
			UNMAP_APAGE(fs, map1, 1);
			goto ret0;
		}
	}
	UNMAP_APAGE(fs, map0, 1);
	UNMAP_APAGE(fs, map1, 1);
	memcpy(&aie, &fs->apage_index[fs->n_active_apages], sizeof(struct apage_index_entry));
	memmove(&fs->apage_index[ap + 2], &fs->apage_index[ap + 1], (fs->n_active_apages - ap - 1) * sizeof(struct apage_index_entry));
	memcpy(&fs->apage_index[ap + 1], &aie, sizeof(struct apage_index_entry));
	fs->apage_index[ap + 1].end_block = (__d_off)fs->apage_index[ap].end_block;
	fs->apage_index[ap].end_block = split_block;
	fs->n_active_apages++;
	return WRITE_APAGE_INDEX(fs);
}

static int WRITE_APAGE_INDEX(SPADFS *fs)
{
	unsigned i, n;
	__d_off sec;
	if (__likely(!CC_CURRENT(fs, fs->a_cc, fs->a_txc))) {
		struct txblock *tx;
		struct buffer_head *bh;
		if (__unlikely(IS_ERR(tx = SPADFS_READ_TX_BLOCK(fs, &bh, "WRITE_APAGE_INDEX 1"))))
			return PTR_ERR(tx);
		start_atomic_buffer_modify(bh);
		CC_SET_CURRENT(fs, &fs->a_cc, &fs->a_txc);
		tx->a_cc = fs->a_cc;
		tx->a_txc = fs->a_txc;
		SPADFS_TX_BLOCK_CHECKSUM(tx);
		end_atomic_buffer_modify(bh);
		__brelse(bh);
	}
	n = APAGE_INDEX_SECTORS(fs->n_apages, fs->blocksize);
	sec = CC_VALID(fs, fs->a_cc, fs->a_txc) ? fs->apage_index0_sec : fs->apage_index1_sec;
	for (i = 0; i < n; i += 1 << fs->sectors_per_block_bits, sec += 1 << fs->sectors_per_block_bits) {
		struct buffer_head *bh;
		void *p = SPADFS_GET_NEW_SECTOR(fs, sec, &bh, "WRITE_APAGE_INDEX 2");
		if (__unlikely(IS_ERR(p))) continue;
		memcpy(p, (__u8 *)fs->apage_index + (i << 9), fs->blocksize);
		mark_buffer_dirty(bh);
		brelse(bh);
	}
	return 0;
}

static int CONVERT_TO_BITMAP(SPADFS *fs, APAGE_MAP *map, __d_off start, __d_off end)
{
	unsigned i;
	COPY_APAGE(fs, fs->tmp_map, map);
	MAKE_APAGE_BITMAP(fs, map, start);
	for (i = sizeof(struct aentry); i < fs->apage_size; i += sizeof(struct aentry)) {
		struct aentry *ae = GET_AENTRY_VALID(fs, fs->tmp_map, i);
		__d_off st;
		unsigned len = ae->len;
		unsigned Xoff, Xlen;
		unsigned shift;
		if (__unlikely(!len)) continue;
		st = ae->start;
		if (__unlikely(st < start) || __unlikely(st + len < st) || __unlikely(st + len > end)) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "POINTER (%LX,%X) OUT OF RANGE (%LX-%LX)", (unsigned long long)st, len, (unsigned long long)start, (unsigned long long)end);
			return -EFSERROR;
		}
		shift = fs->sectors_per_block_bits;
		if (__unlikely(((unsigned)st | (unsigned)len) & ((1 << shift) - 1))) {
			SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "POINTER (%LX,%X) NOT ALIGNED", (unsigned long long)st, len);
		}
		Xoff = (st - start) >> shift;
		Xlen = len >> shift;
		do {
			if (__unlikely(!BMP_TEST(fs, map, Xoff))) {
				SPADFS_ERROR(fs, TXFLAGS_FS_ERROR, "BLOCK %LX+%X FREED TWICE", (unsigned long long)st, (Xoff << shift) - (unsigned)(st - start));
				return -EFSERROR;
			}
			BMP_CLEAR(fs, map, Xoff);
			Xoff++;
		} while (--Xlen);
	}
	return 0;
}

