1
6
7
13
14
17
18
19
20
22
23
24
25
26
27
28
29
30
31
32
33
34
35
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
88
89
90
91
92
95
96
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
163
164
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
222
223
224
225
235
236
237
238
239
240
241
242
243
244
245
246
247
250
251
252
255
256
257
258
259
260
261
262
266
275
276
281
285
286
290
291
292
293
294
295
296
297
298
299
305
306
307
308
309
312
313
316
317
318
319
320
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
377
378
379
383
384
385
386
387
388
389
390
395
396
400
401
407
408
409
410
411
412
421
422
427
428
432
433
434
435
436
437
438
439
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
465
466
467
469
470
471
472
473
474
495
496
499
500
501
502
503
505
506
507
510
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
589
590
591
592
593
594
595
596
597
598
599
600
601
606
607
608
609
610
611
612
616
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
660
661
665
666
667
668
669
670
671
672
673
674
675
679
680
681
682
686
696
697
698
699
704
705
706
707
708
709
710
711
712
/* ... */
#include <string.h>
#include <limits.h>
#include <stdio.h>
#include "tlsf.h"
#include "tlsf_block_functions.h"
#include "tlsf_control_functions.h"6 includes
/* ... */
#define _tlsf_glue2(x, y) x ## y
#define _tlsf_glue(x, y) _tlsf_glue2(x, y)
#define tlsf_static_assert(exp) \
typedef char _tlsf_glue(static_assert, __LINE__) [(exp) ? 1 : -1]...
tlsf_static_assert(sizeof(int) * CHAR_BIT == 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32);
tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64);
static control_t* control_construct(control_t* control, size_t bytes)
{
if (bytes < sizeof(control_t))
{
return NULL;
}{...}
control->fl_index_max = 32 - __builtin_clz(bytes);
if (bytes <= 16 * 1024) control->sl_index_count_log2 = 3;
else if (bytes <= 256 * 1024) control->sl_index_count_log2 = 4;
else control->sl_index_count_log2 = 5;
control->fl_index_shift = (control->sl_index_count_log2 + ALIGN_SIZE_LOG2);
control->sl_index_count = 1 << control->sl_index_count_log2;
control->fl_index_count = control->fl_index_max - control->fl_index_shift + 1;
control->small_block_size = 1 << control->fl_index_shift;
control->size = sizeof(control_t) + (sizeof(*control->sl_bitmap) * control->fl_index_count) +
(sizeof(*control->blocks) * (control->fl_index_count * control->sl_index_count));
if (bytes < control->size + block_size_min)
{
return NULL;
}{...}
control->block_null.next_free = &control->block_null;
control->block_null.prev_free = &control->block_null;
control->fl_bitmap = 0;
control->sl_bitmap = align_ptr(control + 1, sizeof(*control->sl_bitmap));
control->blocks = align_ptr(control->sl_bitmap + control->fl_index_count, sizeof(*control->blocks));
tlsf_assert(sizeof(unsigned int) * CHAR_BIT >= control->sl_index_count
&& "CHAR_BIT less than sl_index_count");
tlsf_assert(ALIGN_SIZE == control->small_block_size / control->sl_index_count);
for (int i = 0; i < control->fl_index_count; ++i)
{
control->sl_bitmap[i] = 0;
for (int j = 0; j < control->sl_index_count; ++j)
{
control->blocks[i * control->sl_index_count + j] = &control->block_null;
}{...}
}{...}
return control;
}{ ... }
/* ... */
typedef struct integrity_t
{
int prev_status;
int status;
}{ ... } integrity_t;
#define tlsf_insist(x) { if (!(x)) { status--; } }
static bool integrity_walker(void* ptr, size_t size, int used, void* user)
{
block_header_t* block = block_from_ptr(ptr);
integrity_t* integ = tlsf_cast(integrity_t*, user);
const int this_prev_status = block_is_prev_free(block) ? 1 : 0;
const int this_status = block_is_free(block) ? 1 : 0;
const size_t this_block_size = block_size(block);
int status = 0;
tlsf_insist(integ->prev_status == this_prev_status && "prev status incorrect");
tlsf_insist(size == this_block_size && "block size incorrect");
if (tlsf_check_hook != NULL)
{
/* ... */
const size_t actual_free_block_size = used ? this_block_size :
this_block_size - offsetof(block_header_t, next_free)- block_header_overhead;
void* ptr_block = used ? (void*)block + block_start_offset :
(void*)block + sizeof(block_header_t);
tlsf_insist(tlsf_check_hook(ptr_block, actual_free_block_size, !used));
}{...}
integ->prev_status = this_status;
integ->status += status;
return true;
}{ ... }
int tlsf_check(tlsf_t tlsf)
{
int i, j;
control_t* control = tlsf_cast(control_t*, tlsf);
int status = 0;
for (i = 0; i < control->fl_index_count; ++i)
{
for (j = 0; j < control->sl_index_count; ++j)
{
const int fl_map = control->fl_bitmap & (1U << i);
const int sl_list = control->sl_bitmap[i];
const int sl_map = sl_list & (1U << j);
const block_header_t* block = control->blocks[i * control->sl_index_count + j];
if (!fl_map)
{
tlsf_insist(!sl_map && "second-level map must be null");
}{...}
if (!sl_map)
{
tlsf_insist(block == &control->block_null && "block list must be null");
continue;
}{...}
tlsf_insist(sl_list && "no free blocks in second-level map");
tlsf_insist(block != &control->block_null && "block should not be null");
while (block != &control->block_null)
{
int fli, sli;
const bool is_block_free = block_is_free(block);
tlsf_insist(is_block_free && "block should be free");
tlsf_insist(!block_is_prev_free(block) && "blocks should have coalesced");
tlsf_insist(!block_is_free(block_next(block)) && "blocks should have coalesced");
tlsf_insist(block_is_prev_free(block_next(block)) && "block should be free");
tlsf_insist(block_size(block) >= block_size_min && "block not minimum size");
mapping_insert(control, block_size(block), &fli, &sli);
tlsf_insist(fli == i && sli == j && "block size indexed in wrong list");
block = block->next_free;
}{...}
}{...}
}{...}
return status;
}{ ... }
#undef tlsf_insist
static bool default_walker(void* ptr, size_t size, int used, void* user)
{
(void)user;
printf("\t%p %s size: %x (%p)\n", ptr, used ? "used" : "free", (unsigned int)size, block_from_ptr(ptr));
return true;
}{ ... }
void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void* user)
{
tlsf_walker pool_walker = walker ? walker : default_walker;
block_header_t* block =
offset_to_block(pool, -(int)block_header_overhead);
bool ret_val = true;
while (block && !block_is_last(block) && ret_val == true)
{
ret_val = pool_walker(
block_to_ptr(block),
block_size(block),
!block_is_free(block),
user);
if (ret_val == true) {
block = block_next(block);
}{...}
}{...}
}{ ... }
size_t tlsf_block_size(void* ptr)
{
size_t size = 0;
if (ptr)
{
const block_header_t* block = block_from_ptr(ptr);
size = block_size(block);
}{...}
return size;
}{ ... }
int tlsf_check_pool(pool_t pool)
{
integrity_t integ = { 0, 0 };
tlsf_walk_pool(pool, integrity_walker, &integ);
return integ.status;
}{ ... }
size_t tlsf_fit_size(tlsf_t tlsf, size_t size)
{
if (size == 0 || tlsf == NULL) {
return 0;
}{...}
control_t* control = tlsf_cast(control_t*, tlsf);
if (size < control->small_block_size) {
return adjust_request_size(tlsf, size, ALIGN_SIZE);
}{...}
size_t sl_interval;
sl_interval = (1 << (32 - __builtin_clz(size) - 1)) / control->sl_index_count;
return size & ~(sl_interval - 1);
}{ ... }
/* ... */
size_t tlsf_size(tlsf_t tlsf)
{
if (tlsf == NULL)
{
return 0;
}{...}
control_t* control = tlsf_cast(control_t*, tlsf);
return control->size;
}{ ... }
/* ... */
size_t tlsf_pool_overhead(void)
{
return 2 * block_header_overhead;
}{ ... }
size_t tlsf_alloc_overhead(void)
{
return block_header_overhead;
}{ ... }
pool_t tlsf_add_pool(tlsf_t tlsf, void* mem, size_t bytes)
{
block_header_t* block;
block_header_t* next;
const size_t pool_overhead = tlsf_pool_overhead();
const size_t pool_bytes = align_down(bytes - pool_overhead, ALIGN_SIZE);
if (((ptrdiff_t)mem % ALIGN_SIZE) != 0)
{
printf("tlsf_add_pool: Memory must be aligned by %u bytes.\n",
(unsigned int)ALIGN_SIZE);
return 0;
}{...}
if (pool_bytes < block_size_min || pool_bytes > tlsf_block_size_max(tlsf))
{
#if defined (TLSF_64BIT)
printf("tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 bytes.\n",
(unsigned int)(pool_overhead + block_size_min),
(unsigned int)((pool_overhead + tlsf_block_size_max(tlsf)) / 256));/* ... */
#else
printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n",
(unsigned int)(pool_overhead + block_size_min),
(unsigned int)(pool_overhead + tlsf_block_size_max(tlsf)));/* ... */
#endif
return 0;
}{...}
/* ... */
block = offset_to_block(mem, -(tlsfptr_t)block_header_overhead);
block_set_size(block, pool_bytes);
block_set_free(block);
block_set_prev_used(block);
block_insert(tlsf_cast(control_t*, tlsf), block);
next = block_link_next(block);
block_set_size(next, 0);
block_set_used(next);
block_set_prev_free(next);
return mem;
}{ ... }
void tlsf_remove_pool(tlsf_t tlsf, pool_t pool)
{
control_t* control = tlsf_cast(control_t*, tlsf);
block_header_t* block = offset_to_block(pool, -(int)block_header_overhead);
int fl = 0, sl = 0;
tlsf_assert(block_is_free(block) && "block should be free");
tlsf_assert(!block_is_free(block_next(block)) && "next block should not be free");
tlsf_assert(block_size(block_next(block)) == 0 && "next block size should be zero");
mapping_insert(control, block_size(block), &fl, &sl);
remove_free_block(control, block, fl, sl);
}{ ... }
/* ... */
#if _DEBUG
int test_ffs_fls()
{
int rv = 0;
rv += (tlsf_ffs(0) == -1) ? 0 : 0x1;
rv += (tlsf_fls(0) == -1) ? 0 : 0x2;
rv += (tlsf_ffs(1) == 0) ? 0 : 0x4;
rv += (tlsf_fls(1) == 0) ? 0 : 0x8;
rv += (tlsf_ffs(0x80000000) == 31) ? 0 : 0x10;
rv += (tlsf_ffs(0x80008000) == 15) ? 0 : 0x20;
rv += (tlsf_fls(0x80000008) == 31) ? 0 : 0x40;
rv += (tlsf_fls(0x7FFFFFFF) == 30) ? 0 : 0x80;
#if defined (TLSF_64BIT)
rv += (tlsf_fls_sizet(0x80000000) == 31) ? 0 : 0x100;
rv += (tlsf_fls_sizet(0x100000000) == 32) ? 0 : 0x200;
rv += (tlsf_fls_sizet(0xffffffffffffffff) == 63) ? 0 : 0x400;/* ... */
#endif
if (rv)
{
printf("test_ffs_fls: %x ffs/fls tests failed.\n", rv);
}{...}
return rv;
}{...}
/* ... */#endif
tlsf_t tlsf_create(void* mem, size_t max_bytes)
{
#if _DEBUG
if (test_ffs_fls())
{
return NULL;
}{...}
#endif/* ... */
if (mem == NULL)
{
return NULL;
}{...}
if (((tlsfptr_t)mem % ALIGN_SIZE) != 0)
{
printf("tlsf_create: Memory must be aligned to %u bytes.\n",
(unsigned int)ALIGN_SIZE);
return NULL;
}{...}
control_t* control_ptr = control_construct(tlsf_cast(control_t*, mem), max_bytes);
return tlsf_cast(tlsf_t, control_ptr);
}{ ... }
tlsf_t tlsf_create_with_pool(void* mem, size_t pool_bytes, size_t max_bytes)
{
tlsf_t tlsf = tlsf_create(mem, max_bytes ? max_bytes : pool_bytes);
if (tlsf != NULL)
{
tlsf_add_pool(tlsf, (char*)mem + tlsf_size(tlsf), pool_bytes - tlsf_size(tlsf));
}{...}
return tlsf;
}{ ... }
void tlsf_destroy(tlsf_t tlsf)
{
(void)tlsf;
}{ ... }
pool_t tlsf_get_pool(tlsf_t tlsf)
{
return tlsf_cast(pool_t, (char*)tlsf + tlsf_size(tlsf));
}{ ... }
void* tlsf_malloc(tlsf_t tlsf, size_t size)
{
control_t* control = tlsf_cast(control_t*, tlsf);
size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
if (adjust == 0) {
return NULL;
}{...}
block_header_t* block = block_locate_free(control, &adjust);
return block_prepare_used(control, block, adjust);
}{ ... }
/* ... */
void* tlsf_malloc_addr(tlsf_t tlsf, size_t size, void *address)
{
control_t* control = tlsf_cast(control_t*, tlsf);
const unsigned int addr_adjusted = align_down(tlsf_cast(unsigned int, address), ALIGN_SIZE);
/* ... */
size_t size_adjusted = align_up(size + (tlsf_cast(unsigned int, address) - addr_adjusted), ALIGN_SIZE);
/* ... */
block_header_t* block = offset_to_block(tlsf_get_pool(tlsf), -(int)block_header_overhead);
const char *alloc_start = tlsf_cast(char*, addr_adjusted);
const char *alloc_end = alloc_start + size_adjusted;
bool block_found = false;
do {
const char *block_start = tlsf_cast(char*, block_to_ptr(block));
const char *block_end = tlsf_cast(char*, block_to_ptr(block)) + block_size(block);
if (block_start <= alloc_start && block_end > alloc_start) {
if (block_end < alloc_end || !block_is_free(block)) {
/* ... */
break;
}{...}
/* ... */
block_found = true;
}{...} else if (!block_is_last(block)) {
block = block_next(block);
}{...}
}{...} while (!block_is_last(block) && block_found == false);
if (!block_found) {
return NULL;
}{...}
block_remove(control, block);
/* ... */
const size_t space_before_addr_adjusted = addr_adjusted - tlsf_cast(unsigned int, block_to_ptr(block));
block_header_t *return_block = block;
if (space_before_addr_adjusted >= block_size_min) {
return_block = block_trim_free_leading(control, block, space_before_addr_adjusted);
}{...}
else {
size_adjusted += space_before_addr_adjusted;
}{...}
return block_prepare_used(control, return_block, size_adjusted);
}{ ... }
/* ... */
void* tlsf_memalign_offs(tlsf_t tlsf, size_t align, size_t size, size_t data_offset)
{
control_t* control = tlsf_cast(control_t*, tlsf);
const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
const size_t off_adjust = align_up(data_offset, ALIGN_SIZE);
/* ... */
const size_t gap_minimum = sizeof(block_header_t) + off_adjust;
/* ... */
const size_t size_with_gap = adjust_request_size(tlsf, adjust + align + gap_minimum - off_adjust, align);
/* ... */
size_t aligned_size = (adjust && align > ALIGN_SIZE) ? size_with_gap : adjust;
block_header_t* block = block_locate_free(control, &aligned_size);
tlsf_assert(sizeof(block_header_t) == block_size_min + block_header_overhead);
if (block)
{
void* ptr = block_to_ptr(block);
void* aligned = align_ptr(ptr, align);
size_t gap = tlsf_cast(size_t,
tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));
/* ... */
if ((gap && gap < gap_minimum) || (!gap && off_adjust && align > ALIGN_SIZE))
{
const size_t gap_remain = gap_minimum - gap;
const size_t offset = tlsf_max(gap_remain, align);
const void* next_aligned = tlsf_cast(void*,
tlsf_cast(tlsfptr_t, aligned) + offset);
aligned = align_ptr(next_aligned, align);
gap = tlsf_cast(size_t,
tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));
}{...}
if (gap)
{
tlsf_assert(gap >= gap_minimum && "gap size too small");
block = block_trim_free_leading(control, block, gap - off_adjust);
}{...}
}{...}
return block_prepare_used(control, block, adjust);
}{ ... }
/* ... */
void* tlsf_memalign(tlsf_t tlsf, size_t align, size_t size)
{
return tlsf_memalign_offs(tlsf, align, size, 0);
}{ ... }
void tlsf_free(tlsf_t tlsf, void* ptr)
{
if (ptr)
{
control_t* control = tlsf_cast(control_t*, tlsf);
block_header_t* block = block_from_ptr(ptr);
tlsf_assert(!block_is_free(block) && "block already marked as free");
block_mark_as_free(block);
block = block_merge_prev(control, block);
block = block_merge_next(control, block);
block_insert(control, block);
}{...}
}{ ... }
/* ... */
void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size)
{
control_t* control = tlsf_cast(control_t*, tlsf);
void* p = 0;
if (ptr && size == 0)
{
tlsf_free(tlsf, ptr);
}{...}
else if (!ptr)
{
p = tlsf_malloc(tlsf, size);
}{...}
else
{
block_header_t* block = block_from_ptr(ptr);
block_header_t* next = block_next(block);
const size_t cursize = block_size(block);
const size_t combined = cursize + block_size(next) + block_header_overhead;
const size_t adjust = adjust_request_size(tlsf, size, ALIGN_SIZE);
if (adjust == 0)
{
return p;
}{...}
tlsf_assert(!block_is_free(block) && "block already marked as free");
/* ... */
if (adjust > cursize && (!block_is_free(next) || adjust > combined))
{
p = tlsf_malloc(tlsf, size);
if (p)
{
const size_t minsize = tlsf_min(cursize, size);
memcpy(p, ptr, minsize);
tlsf_free(tlsf, ptr);
}{...}
}{...}
else
{
if (adjust > cursize)
{
block_merge_next(control, block);
block_mark_as_used(block);
}{...}
block_trim_used(control, block, adjust);
p = ptr;
}{...}
}{...}
return p;
}{ ... }