Ruby: st.c Source File

Ruby 3.0.5p211 (2022-11-24 revision ba5cf0f7c52d4d35cc6a173c89eda98ceffa2dcf)

st.c
Go to the documentation of this file.
1/* This is a public domain general purpose hash table package
2 originally written by Peter Moore @ UCB.
3
4 The hash table data structures were redesigned and the package was
5 rewritten by Vladimir Makarov <vmakarov@redhat.com>. */
6
7/* The original package implemented classic bucket-based hash tables
8 with entries doubly linked for an access by their insertion order.
9 To decrease pointer chasing and as a consequence to improve a data
10 locality the current implementation is based on storing entries in
11 an array and using hash tables with open addressing. The current
12 entries are more compact in comparison with the original ones and
13 this also improves the data locality.
14
15 The hash table has two arrays called *bins* and *entries*.
16
17 bins:
18 -------
19 | | entries array:
20 |-------| --------------------------------
21 | index | | | entry: | | |
22 |-------| | | | | |
23 | ... | | ... | hash | ... | ... |
24 |-------| | | key | | |
25 | empty | | | record | | |
26 |-------| --------------------------------
27 | ... | ^ ^
28 |-------| |_ entries start |_ entries bound
29 |deleted|
30 -------
31
32 o The entry array contains table entries in the same order as they
33 were inserted.
34
35 When the first entry is deleted, a variable containing index of
36 the current first entry (*entries start*) is changed. In all
37 other cases of the deletion, we just mark the entry as deleted by
38 using a reserved hash value.
39
40 Such organization of the entry storage makes operations of the
41 table shift and the entries traversal very fast.
42
43 o The bins provide access to the entries by their keys. The
44 key hash is mapped to a bin containing *index* of the
45 corresponding entry in the entry array.
46
47 The bin array size is always power of two, it makes mapping very
48 fast by using the corresponding lower bits of the hash.
49 Generally it is not a good idea to ignore some part of the hash.
50 But alternative approach is worse. For example, we could use a
51 modulo operation for mapping and a prime number for the size of
52 the bin array. Unfortunately, the modulo operation for big
53 64-bit numbers are extremely slow (it takes more than 100 cycles
54 on modern Intel CPUs).
55
56 Still other bits of the hash value are used when the mapping
57 results in a collision. In this case we use a secondary hash
58 value which is a result of a function of the collision bin
59 index and the original hash value. The function choice
60 guarantees that we can traverse all bins and finally find the
61 corresponding bin as after several iterations the function
62 becomes a full cycle linear congruential generator because it
63 satisfies requirements of the Hull-Dobell theorem.
64
65 When an entry is removed from the table besides marking the
66 hash in the corresponding entry described above, we also mark
67 the bin by a special value in order to find entries which had
68 a collision with the removed entries.
69
70 There are two reserved values for the bins. One denotes an
71 empty bin, another one denotes a bin for a deleted entry.
72
73 o The length of the bin array is at least two times more than the
74 entry array length. This keeps the table load factor healthy.
75 The trigger of rebuilding the table is always a case when we can
76 not insert an entry anymore at the entries bound. We could
77 change the entries bound too in case of deletion but than we need
78 a special code to count bins with corresponding deleted entries
79 and reset the bin values when there are too many bins
80 corresponding deleted entries
81
82 Table rebuilding is done by creation of a new entry array and
83 bins of an appropriate size. We also try to reuse the arrays
84 in some cases by compacting the array and removing deleted
85 entries.
86
87 o To save memory very small tables have no allocated arrays
88 bins. We use a linear search for an access by a key.
89
90 o To save more memory we use 8-, 16-, 32- and 64- bit indexes in
91 bins depending on the current hash table size.
92
93 o The implementation takes into account that the table can be
94 rebuilt during hashing or comparison functions. It can happen if
95 the functions are implemented in Ruby and a thread switch occurs
96 during their execution.
97
98 This implementation speeds up the Ruby hash table benchmarks in
99 average by more 40% on Intel Haswell CPU.
100
101*/
102
103#ifdef NOT_RUBY
104#include "regint.h"
105#include "st.h"
106#else
107#include "internal.h"
108#include "internal/bits.h"
109#include "internal/hash.h"
110#include "internal/sanitizers.h"
111#endif
112
113#include <stdio.h>
114#ifdef HAVE_STDLIB_H
115#include <stdlib.h>
116#endif
117#include <string.h>
118#include <assert.h>
119
120#ifdef __GNUC__
121#define PREFETCH(addr, write_p) __builtin_prefetch(addr, write_p)
122#define EXPECT(expr, val) __builtin_expect(expr, val)
123#define ATTRIBUTE_UNUSED __attribute__((unused))
124#else
125#define PREFETCH(addr, write_p)
126#define EXPECT(expr, val) (expr)
127#define ATTRIBUTE_UNUSED
128#endif
129
130/* The type of hashes. */
131 typedef st_index_t st_hash_t;
132
133 struct st_table_entry {
134 st_hash_t hash;
135 st_data_t key;
136 st_data_t record;
137};
138
139#define type_numhash st_hashtype_num
140static const struct st_hash_type st_hashtype_num = {
141 st_numcmp,
142 st_numhash,
143};
144
145static int st_strcmp(st_data_t, st_data_t);
146static st_index_t strhash(st_data_t);
147static const struct st_hash_type type_strhash = {
148 st_strcmp,
149 strhash,
150};
151
152static int st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs);
153static st_index_t strcasehash(st_data_t);
154static const struct st_hash_type type_strcasehash = {
155 st_locale_insensitive_strcasecmp_i,
156 strcasehash,
157};
158
159/* Value used to catch uninitialized entries/bins during debugging.
160 There is a possibility for a false alarm, but its probability is
161 extremely small. */
162#define ST_INIT_VAL 0xafafafafafafafaf
163#define ST_INIT_VAL_BYTE 0xafa
164
165#ifdef RUBY
166#undef malloc
167#undef realloc
168#undef calloc
169#undef free
170#define malloc ruby_xmalloc
171#define calloc ruby_xcalloc
172#define realloc ruby_xrealloc
173#define free ruby_xfree
174#endif
175
176#define EQUAL(tab,x,y) ((x) == (y) || (*(tab)->type->compare)((x),(y)) == 0)
177#define PTR_EQUAL(tab, ptr, hash_val, key_) \
178 ((ptr)->hash == (hash_val) && EQUAL((tab), (key_), (ptr)->key))
179
180/* As PRT_EQUAL only its result is returned in RES. REBUILT_P is set
181 up to TRUE if the table is rebuilt during the comparison. */
182#define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p) \
183 do { \
184 unsigned int _old_rebuilds_num = (tab)->rebuilds_num; \
185 res = PTR_EQUAL(tab, ptr, hash_val, key); \
186 rebuilt_p = _old_rebuilds_num != (tab)->rebuilds_num; \
187 } while (FALSE)
188
189/* Features of a table. */
190 struct st_features {
191 /* Power of 2 used for number of allocated entries. */
192 unsigned char entry_power;
193 /* Power of 2 used for number of allocated bins. Depending on the
194 table size, the number of bins is 2-4 times more than the
195 number of entries. */
196 unsigned char bin_power;
197 /* Enumeration of sizes of bins (8-bit, 16-bit etc). */
198 unsigned char size_ind;
199 /* Bins are packed in words of type st_index_t. The following is
200 a size of bins counted by words. */
201 st_index_t bins_words;
202};
203
204/* Features of all possible size tables. */
205#if SIZEOF_ST_INDEX_T == 8
206#define MAX_POWER2 62
207static const struct st_features features[] = {
208 {0, 1, 0, 0x0},
209 {1, 2, 0, 0x1},
210 {2, 3, 0, 0x1},
211 {3, 4, 0, 0x2},
212 {4, 5, 0, 0x4},
213 {5, 6, 0, 0x8},
214 {6, 7, 0, 0x10},
215 {7, 8, 0, 0x20},
216 {8, 9, 1, 0x80},
217 {9, 10, 1, 0x100},
218 {10, 11, 1, 0x200},
219 {11, 12, 1, 0x400},
220 {12, 13, 1, 0x800},
221 {13, 14, 1, 0x1000},
222 {14, 15, 1, 0x2000},
223 {15, 16, 1, 0x4000},
224 {16, 17, 2, 0x10000},
225 {17, 18, 2, 0x20000},
226 {18, 19, 2, 0x40000},
227 {19, 20, 2, 0x80000},
228 {20, 21, 2, 0x100000},
229 {21, 22, 2, 0x200000},
230 {22, 23, 2, 0x400000},
231 {23, 24, 2, 0x800000},
232 {24, 25, 2, 0x1000000},
233 {25, 26, 2, 0x2000000},
234 {26, 27, 2, 0x4000000},
235 {27, 28, 2, 0x8000000},
236 {28, 29, 2, 0x10000000},
237 {29, 30, 2, 0x20000000},
238 {30, 31, 2, 0x40000000},
239 {31, 32, 2, 0x80000000},
240 {32, 33, 3, 0x200000000},
241 {33, 34, 3, 0x400000000},
242 {34, 35, 3, 0x800000000},
243 {35, 36, 3, 0x1000000000},
244 {36, 37, 3, 0x2000000000},
245 {37, 38, 3, 0x4000000000},
246 {38, 39, 3, 0x8000000000},
247 {39, 40, 3, 0x10000000000},
248 {40, 41, 3, 0x20000000000},
249 {41, 42, 3, 0x40000000000},
250 {42, 43, 3, 0x80000000000},
251 {43, 44, 3, 0x100000000000},
252 {44, 45, 3, 0x200000000000},
253 {45, 46, 3, 0x400000000000},
254 {46, 47, 3, 0x800000000000},
255 {47, 48, 3, 0x1000000000000},
256 {48, 49, 3, 0x2000000000000},
257 {49, 50, 3, 0x4000000000000},
258 {50, 51, 3, 0x8000000000000},
259 {51, 52, 3, 0x10000000000000},
260 {52, 53, 3, 0x20000000000000},
261 {53, 54, 3, 0x40000000000000},
262 {54, 55, 3, 0x80000000000000},
263 {55, 56, 3, 0x100000000000000},
264 {56, 57, 3, 0x200000000000000},
265 {57, 58, 3, 0x400000000000000},
266 {58, 59, 3, 0x800000000000000},
267 {59, 60, 3, 0x1000000000000000},
268 {60, 61, 3, 0x2000000000000000},
269 {61, 62, 3, 0x4000000000000000},
270 {62, 63, 3, 0x8000000000000000},
271};
272
273#else
274#define MAX_POWER2 30
275
276static const struct st_features features[] = {
277 {0, 1, 0, 0x1},
278 {1, 2, 0, 0x1},
279 {2, 3, 0, 0x2},
280 {3, 4, 0, 0x4},
281 {4, 5, 0, 0x8},
282 {5, 6, 0, 0x10},
283 {6, 7, 0, 0x20},
284 {7, 8, 0, 0x40},
285 {8, 9, 1, 0x100},
286 {9, 10, 1, 0x200},
287 {10, 11, 1, 0x400},
288 {11, 12, 1, 0x800},
289 {12, 13, 1, 0x1000},
290 {13, 14, 1, 0x2000},
291 {14, 15, 1, 0x4000},
292 {15, 16, 1, 0x8000},
293 {16, 17, 2, 0x20000},
294 {17, 18, 2, 0x40000},
295 {18, 19, 2, 0x80000},
296 {19, 20, 2, 0x100000},
297 {20, 21, 2, 0x200000},
298 {21, 22, 2, 0x400000},
299 {22, 23, 2, 0x800000},
300 {23, 24, 2, 0x1000000},
301 {24, 25, 2, 0x2000000},
302 {25, 26, 2, 0x4000000},
303 {26, 27, 2, 0x8000000},
304 {27, 28, 2, 0x10000000},
305 {28, 29, 2, 0x20000000},
306 {29, 30, 2, 0x40000000},
307 {30, 31, 2, 0x80000000},
308};
309
310#endif
311
312/* The reserved hash value and its substitution. */
313#define RESERVED_HASH_VAL (~(st_hash_t) 0)
314#define RESERVED_HASH_SUBSTITUTION_VAL ((st_hash_t) 0)
315
316/* Return hash value of KEY for table TAB. */
317static inline st_hash_t
318do_hash(st_data_t key, st_table *tab)
319{
320 st_hash_t hash = (st_hash_t)(tab->type->hash)(key);
321
322 /* RESERVED_HASH_VAL is used for a deleted entry. Map it into
323 another value. Such mapping should be extremely rare. */
324 return hash == RESERVED_HASH_VAL ? RESERVED_HASH_SUBSTITUTION_VAL : hash;
325}
326
327/* Power of 2 defining the minimal number of allocated entries. */
328#define MINIMAL_POWER2 2
329
330#if MINIMAL_POWER2 < 2
331#error "MINIMAL_POWER2 should be >= 2"
332#endif
333
334/* If the power2 of the allocated `entries` is less than the following
335 value, don't allocate bins and use a linear search. */
336#define MAX_POWER2_FOR_TABLES_WITHOUT_BINS 4
337
338/* Return smallest n >= MINIMAL_POWER2 such 2^n > SIZE. */
339static int
340get_power2(st_index_t size)
341{
342 unsigned int n = ST_INDEX_BITS - nlz_intptr(size);
343 if (n <= MAX_POWER2)
344 return n < MINIMAL_POWER2 ? MINIMAL_POWER2 : n;
345#ifndef NOT_RUBY
346 /* Ran out of the table entries */
347 rb_raise(rb_eRuntimeError, "st_table too big");
348#endif
349 /* should raise exception */
350 return -1;
351}
352
353/* Return value of N-th bin in array BINS of table with bins size
354 index S. */
355static inline st_index_t
356get_bin(st_index_t *bins, int s, st_index_t n)
357{
358 return (s == 0 ? ((unsigned char *) bins)[n]
359 : s == 1 ? ((unsigned short *) bins)[n]
360 : s == 2 ? ((unsigned int *) bins)[n]
361 : ((st_index_t *) bins)[n]);
362}
363
364/* Set up N-th bin in array BINS of table with bins size index S to
365 value V. */
366static inline void
367set_bin(st_index_t *bins, int s, st_index_t n, st_index_t v)
368{
369 if (s == 0) ((unsigned char *) bins)[n] = (unsigned char) v;
370 else if (s == 1) ((unsigned short *) bins)[n] = (unsigned short) v;
371 else if (s == 2) ((unsigned int *) bins)[n] = (unsigned int) v;
372 else ((st_index_t *) bins)[n] = v;
373}
374
375/* These macros define reserved values for empty table bin and table
376 bin which contains a deleted entry. We will never use such values
377 for an entry index in bins. */
378#define EMPTY_BIN 0
379#define DELETED_BIN 1
380/* Base of a real entry index in the bins. */
381#define ENTRY_BASE 2
382
383/* Mark I-th bin of table TAB as empty, in other words not
384 corresponding to any entry. */
385#define MARK_BIN_EMPTY(tab, i) (set_bin((tab)->bins, get_size_ind(tab), i, EMPTY_BIN))
386
387/* Values used for not found entry and bin with given
388 characteristics. */
389#define UNDEFINED_ENTRY_IND (~(st_index_t) 0)
390#define UNDEFINED_BIN_IND (~(st_index_t) 0)
391
392/* Entry and bin values returned when we found a table rebuild during
393 the search. */
394#define REBUILT_TABLE_ENTRY_IND (~(st_index_t) 1)
395#define REBUILT_TABLE_BIN_IND (~(st_index_t) 1)
396
397/* Mark I-th bin of table TAB as corresponding to a deleted table
398 entry. Update number of entries in the table and number of bins
399 corresponding to deleted entries. */
400#define MARK_BIN_DELETED(tab, i) \
401 do { \
402 set_bin((tab)->bins, get_size_ind(tab), i, DELETED_BIN); \
403 } while (0)
404
405/* Macros to check that value B is used empty bins and bins
406 corresponding deleted entries. */
407#define EMPTY_BIN_P(b) ((b) == EMPTY_BIN)
408#define DELETED_BIN_P(b) ((b) == DELETED_BIN)
409#define EMPTY_OR_DELETED_BIN_P(b) ((b) <= DELETED_BIN)
410
411/* Macros to check empty bins and bins corresponding to deleted
412 entries. Bins are given by their index I in table TAB. */
413#define IND_EMPTY_BIN_P(tab, i) (EMPTY_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
414#define IND_DELETED_BIN_P(tab, i) (DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
415#define IND_EMPTY_OR_DELETED_BIN_P(tab, i) (EMPTY_OR_DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
416
417/* Macros for marking and checking deleted entries given by their
418 pointer E_PTR. */
419#define MARK_ENTRY_DELETED(e_ptr) ((e_ptr)->hash = RESERVED_HASH_VAL)
420#define DELETED_ENTRY_P(e_ptr) ((e_ptr)->hash == RESERVED_HASH_VAL)
421
422/* Return bin size index of table TAB. */
423static inline unsigned int
424get_size_ind(const st_table *tab)
425{
426 return tab->size_ind;
427}
428
429/* Return the number of allocated bins of table TAB. */
430static inline st_index_t
431get_bins_num(const st_table *tab)
432{
433 return ((st_index_t) 1)<<tab->bin_power;
434}
435
436/* Return mask for a bin index in table TAB. */
437static inline st_index_t
438bins_mask(const st_table *tab)
439{
440 return get_bins_num(tab) - 1;
441}
442
443/* Return the index of table TAB bin corresponding to
444 HASH_VALUE. */
445static inline st_index_t
446hash_bin(st_hash_t hash_value, st_table *tab)
447{
448 return hash_value & bins_mask(tab);
449}
450
451/* Return the number of allocated entries of table TAB. */
452static inline st_index_t
453get_allocated_entries(const st_table *tab)
454{
455 return ((st_index_t) 1)<<tab->entry_power;
456}
457
458/* Return size of the allocated bins of table TAB. */
459static inline st_index_t
460bins_size(const st_table *tab)
461{
462 return features[tab->entry_power].bins_words * sizeof (st_index_t);
463}
464
465/* Mark all bins of table TAB as empty. */
466static void
467initialize_bins(st_table *tab)
468{
469 memset(tab->bins, 0, bins_size(tab));
470}
471
472/* Make table TAB empty. */
473static void
474make_tab_empty(st_table *tab)
475{
476 tab->num_entries = 0;
477 tab->entries_start = tab->entries_bound = 0;
478 if (tab->bins != NULL)
479 initialize_bins(tab);
480}
481
482#ifdef HASH_LOG
483#ifdef HAVE_UNISTD_H
484#include <unistd.h>
485#endif
486static struct {
487 int all, total, num, str, strcase;
488} collision;
489
490/* Flag switching off output of package statistics at the end of
491 program. */
492static int init_st = 0;
493
494/* Output overall number of table searches and collisions into a
495 temporary file. */
496static void
497stat_col(void)
498{
499 char fname[10+sizeof(long)*3];
500 FILE *f;
501 if (!collision.total) return;
502 f = fopen((snprintf(fname, sizeof(fname), "/tmp/col%ld", (long)getpid()), fname), "w");
503 if (f == NULL)
504 return;
505 fprintf(f, "collision: %d / %d (%6.2f)\n", collision.all, collision.total,
506 ((double)collision.all / (collision.total)) * 100);
507 fprintf(f, "num: %d, str: %d, strcase: %d\n", collision.num, collision.str, collision.strcase);
508 fclose(f);
509}
510#endif
511
512/* Create and return table with TYPE which can hold at least SIZE
513 entries. The real number of entries which the table can hold is
514 the nearest power of two for SIZE. */
515 st_table *
516 st_init_table_with_size(const struct st_hash_type *type, st_index_t size)
517{
518 st_table *tab;
519 int n;
520
521#ifdef HASH_LOG
522#if HASH_LOG+0 < 0
523 {
524 const char *e = getenv("ST_HASH_LOG");
525 if (!e || !*e) init_st = 1;
526 }
527#endif
528 if (init_st == 0) {
529 init_st = 1;
530 atexit(stat_col);
531 }
532#endif
533
534 n = get_power2(size);
535#ifndef RUBY
536 if (n < 0)
537 return NULL;
538#endif
539 tab = (st_table *) malloc(sizeof (st_table));
540#ifndef RUBY
541 if (tab == NULL)
542 return NULL;
543#endif
544 tab->type = type;
545 tab->entry_power = n;
546 tab->bin_power = features[n].bin_power;
547 tab->size_ind = features[n].size_ind;
548 if (n <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
549 tab->bins = NULL;
550 else {
551 tab->bins = (st_index_t *) malloc(bins_size(tab));
552#ifndef RUBY
553 if (tab->bins == NULL) {
554 free(tab);
555 return NULL;
556 }
557#endif
558 }
559 tab->entries = (st_table_entry *) malloc(get_allocated_entries(tab)
560 * sizeof(st_table_entry));
561#ifndef RUBY
562 if (tab->entries == NULL) {
563 st_free_table(tab);
564 return NULL;
565 }
566#endif
567 make_tab_empty(tab);
568 tab->rebuilds_num = 0;
569 return tab;
570}
571
572/* Create and return table with TYPE which can hold a minimal number
573 of entries (see comments for get_power2). */
574 st_table *
575 st_init_table(const struct st_hash_type *type)
576{
577 return st_init_table_with_size(type, 0);
578}
579
580/* Create and return table which can hold a minimal number of
581 numbers. */
582 st_table *
583st_init_numtable(void)
584{
585 return st_init_table(&type_numhash);
586}
587
588/* Create and return table which can hold SIZE numbers. */
589 st_table *
590 st_init_numtable_with_size(st_index_t size)
591{
592 return st_init_table_with_size(&type_numhash, size);
593}
594
595/* Create and return table which can hold a minimal number of
596 strings. */
597 st_table *
598st_init_strtable(void)
599{
600 return st_init_table(&type_strhash);
601}
602
603/* Create and return table which can hold SIZE strings. */
604 st_table *
605 st_init_strtable_with_size(st_index_t size)
606{
607 return st_init_table_with_size(&type_strhash, size);
608}
609
610/* Create and return table which can hold a minimal number of strings
611 whose character case is ignored. */
612 st_table *
613st_init_strcasetable(void)
614{
615 return st_init_table(&type_strcasehash);
616}
617
618/* Create and return table which can hold SIZE strings whose character
619 case is ignored. */
620 st_table *
621 st_init_strcasetable_with_size(st_index_t size)
622{
623 return st_init_table_with_size(&type_strcasehash, size);
624}
625
626/* Make table TAB empty. */
627void
628 st_clear(st_table *tab)
629{
630 make_tab_empty(tab);
631 tab->rebuilds_num++;
632}
633
634/* Free table TAB space. */
635void
636 st_free_table(st_table *tab)
637{
638 if (tab->bins != NULL)
639 free(tab->bins);
640 free(tab->entries);
641 free(tab);
642}
643
644/* Return byte size of memory allocated for table TAB. */
645size_t
646 st_memsize(const st_table *tab)
647{
648 return(sizeof(st_table)
649 + (tab->bins == NULL ? 0 : bins_size(tab))
650 + get_allocated_entries(tab) * sizeof(st_table_entry));
651}
652
653static st_index_t
654find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key);
655
656static st_index_t
657find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key);
658
659static st_index_t
660find_table_bin_ind_direct(st_table *table, st_hash_t hash_value, st_data_t key);
661
662static st_index_t
663find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
664 st_data_t key, st_index_t *bin_ind);
665
666#ifdef HASH_LOG
667static void
668count_collision(const struct st_hash_type *type)
669{
670 collision.all++;
671 if (type == &type_numhash) {
672 collision.num++;
673 }
674 else if (type == &type_strhash) {
675 collision.strcase++;
676 }
677 else if (type == &type_strcasehash) {
678 collision.str++;
679 }
680}
681
682#define COLLISION (collision_check ? count_collision(tab->type) : (void)0)
683#define FOUND_BIN (collision_check ? collision.total++ : (void)0)
684#define collision_check 0
685#else
686#define COLLISION
687#define FOUND_BIN
688#endif
689
690/* If the number of entries in the table is at least REBUILD_THRESHOLD
691 times less than the entry array length, decrease the table
692 size. */
693#define REBUILD_THRESHOLD 4
694
695#if REBUILD_THRESHOLD < 2
696#error "REBUILD_THRESHOLD should be >= 2"
697#endif
698
699/* Rebuild table TAB. Rebuilding removes all deleted bins and entries
700 and can change size of the table entries and bins arrays.
701 Rebuilding is implemented by creation of a new table or by
702 compaction of the existing one. */
703static void
704rebuild_table(st_table *tab)
705{
706 st_index_t i, ni, bound;
707 unsigned int size_ind;
708 st_table *new_tab;
709 st_table_entry *entries, *new_entries;
710 st_table_entry *curr_entry_ptr;
711 st_index_t *bins;
712 st_index_t bin_ind;
713
714 bound = tab->entries_bound;
715 entries = tab->entries;
716 if ((2 * tab->num_entries <= get_allocated_entries(tab)
717 && REBUILD_THRESHOLD * tab->num_entries > get_allocated_entries(tab))
718 || tab->num_entries < (1 << MINIMAL_POWER2)) {
719 /* Compaction: */
720 tab->num_entries = 0;
721 if (tab->bins != NULL)
722 initialize_bins(tab);
723 new_tab = tab;
724 new_entries = entries;
725 }
726 else {
727 new_tab = st_init_table_with_size(tab->type,
728 2 * tab->num_entries - 1);
729 new_entries = new_tab->entries;
730 }
731 ni = 0;
732 bins = new_tab->bins;
733 size_ind = get_size_ind(new_tab);
734 for (i = tab->entries_start; i < bound; i++) {
735 curr_entry_ptr = &entries[i];
736 PREFETCH(entries + i + 1, 0);
737 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
738 continue;
739 if (&new_entries[ni] != curr_entry_ptr)
740 new_entries[ni] = *curr_entry_ptr;
741 if (EXPECT(bins != NULL, 1)) {
742 bin_ind = find_table_bin_ind_direct(new_tab, curr_entry_ptr->hash,
743 curr_entry_ptr->key);
744 set_bin(bins, size_ind, bin_ind, ni + ENTRY_BASE);
745 }
746 new_tab->num_entries++;
747 ni++;
748 }
749 if (new_tab != tab) {
750 tab->entry_power = new_tab->entry_power;
751 tab->bin_power = new_tab->bin_power;
752 tab->size_ind = new_tab->size_ind;
753 if (tab->bins != NULL)
754 free(tab->bins);
755 tab->bins = new_tab->bins;
756 free(tab->entries);
757 tab->entries = new_tab->entries;
758 free(new_tab);
759 }
760 tab->entries_start = 0;
761 tab->entries_bound = tab->num_entries;
762 tab->rebuilds_num++;
763}
764
765/* Return the next secondary hash index for table TAB using previous
766 index IND and PERTERB. Finally modulo of the function becomes a
767 full *cycle linear congruential generator*, in other words it
768 guarantees traversing all table bins in extreme case.
769
770 According the Hull-Dobell theorem a generator
771 "Xnext = (a*Xprev + c) mod m" is a full cycle generator iff
772 o m and c are relatively prime
773 o a-1 is divisible by all prime factors of m
774 o a-1 is divisible by 4 if m is divisible by 4.
775
776 For our case a is 5, c is 1, and m is a power of two. */
777static inline st_index_t
778secondary_hash(st_index_t ind, st_table *tab, st_index_t *perterb)
779{
780 *perterb >>= 11;
781 ind = (ind << 2) + ind + *perterb + 1;
782 return hash_bin(ind, tab);
783}
784
785/* Find an entry with HASH_VALUE and KEY in TABLE using a linear
786 search. Return the index of the found entry in array `entries`.
787 If it is not found, return UNDEFINED_ENTRY_IND. If the table was
788 rebuilt during the search, return REBUILT_TABLE_ENTRY_IND. */
789static inline st_index_t
790find_entry(st_table *tab, st_hash_t hash_value, st_data_t key)
791{
792 int eq_p, rebuilt_p;
793 st_index_t i, bound;
794 st_table_entry *entries;
795
796 bound = tab->entries_bound;
797 entries = tab->entries;
798 for (i = tab->entries_start; i < bound; i++) {
799 DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
800 if (EXPECT(rebuilt_p, 0))
801 return REBUILT_TABLE_ENTRY_IND;
802 if (eq_p)
803 return i;
804 }
805 return UNDEFINED_ENTRY_IND;
806}
807
808/* Use the quadratic probing. The method has a better data locality
809 but more collisions than the current approach. In average it
810 results in a bit slower search. */
811/*#define QUADRATIC_PROBE*/
812
813/* Return index of entry with HASH_VALUE and KEY in table TAB. If
814 there is no such entry, return UNDEFINED_ENTRY_IND. If the table
815 was rebuilt during the search, return REBUILT_TABLE_ENTRY_IND. */
816static st_index_t
817find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
818{
819 int eq_p, rebuilt_p;
820 st_index_t ind;
821#ifdef QUADRATIC_PROBE
822 st_index_t d;
823#else
824 st_index_t peterb;
825#endif
826 st_index_t bin;
827 st_table_entry *entries = tab->entries;
828
829 ind = hash_bin(hash_value, tab);
830#ifdef QUADRATIC_PROBE
831 d = 1;
832#else
833 peterb = hash_value;
834#endif
835 FOUND_BIN;
836 for (;;) {
837 bin = get_bin(tab->bins, get_size_ind(tab), ind);
838 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
839 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
840 if (EXPECT(rebuilt_p, 0))
841 return REBUILT_TABLE_ENTRY_IND;
842 if (eq_p)
843 break;
844 } else if (EMPTY_BIN_P(bin))
845 return UNDEFINED_ENTRY_IND;
846#ifdef QUADRATIC_PROBE
847 ind = hash_bin(ind + d, tab);
848 d++;
849#else
850 ind = secondary_hash(ind, tab, &peterb);
851#endif
852 COLLISION;
853 }
854 return bin;
855}
856
857/* Find and return index of table TAB bin corresponding to an entry
858 with HASH_VALUE and KEY. If there is no such bin, return
859 UNDEFINED_BIN_IND. If the table was rebuilt during the search,
860 return REBUILT_TABLE_BIN_IND. */
861static st_index_t
862find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
863{
864 int eq_p, rebuilt_p;
865 st_index_t ind;
866#ifdef QUADRATIC_PROBE
867 st_index_t d;
868#else
869 st_index_t peterb;
870#endif
871 st_index_t bin;
872 st_table_entry *entries = tab->entries;
873
874 ind = hash_bin(hash_value, tab);
875#ifdef QUADRATIC_PROBE
876 d = 1;
877#else
878 peterb = hash_value;
879#endif
880 FOUND_BIN;
881 for (;;) {
882 bin = get_bin(tab->bins, get_size_ind(tab), ind);
883 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
884 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
885 if (EXPECT(rebuilt_p, 0))
886 return REBUILT_TABLE_BIN_IND;
887 if (eq_p)
888 break;
889 } else if (EMPTY_BIN_P(bin))
890 return UNDEFINED_BIN_IND;
891#ifdef QUADRATIC_PROBE
892 ind = hash_bin(ind + d, tab);
893 d++;
894#else
895 ind = secondary_hash(ind, tab, &peterb);
896#endif
897 COLLISION;
898 }
899 return ind;
900}
901
902/* Find and return index of table TAB bin corresponding to an entry
903 with HASH_VALUE and KEY. The entry should be in the table
904 already. */
905static st_index_t
906find_table_bin_ind_direct(st_table *tab, st_hash_t hash_value, st_data_t key)
907{
908 st_index_t ind;
909#ifdef QUADRATIC_PROBE
910 st_index_t d;
911#else
912 st_index_t peterb;
913#endif
914 st_index_t bin;
915
916 ind = hash_bin(hash_value, tab);
917#ifdef QUADRATIC_PROBE
918 d = 1;
919#else
920 peterb = hash_value;
921#endif
922 FOUND_BIN;
923 for (;;) {
924 bin = get_bin(tab->bins, get_size_ind(tab), ind);
925 if (EMPTY_OR_DELETED_BIN_P(bin))
926 return ind;
927#ifdef QUADRATIC_PROBE
928 ind = hash_bin(ind + d, tab);
929 d++;
930#else
931 ind = secondary_hash(ind, tab, &peterb);
932#endif
933 COLLISION;
934 }
935}
936
937/* Return index of table TAB bin for HASH_VALUE and KEY through
938 BIN_IND and the pointed value as the function result. Reserve the
939 bin for inclusion of the corresponding entry into the table if it
940 is not there yet. We always find such bin as bins array length is
941 bigger entries array. Although we can reuse a deleted bin, the
942 result bin value is always empty if the table has no entry with
943 KEY. Return the entries array index of the found entry or
944 UNDEFINED_ENTRY_IND if it is not found. If the table was rebuilt
945 during the search, return REBUILT_TABLE_ENTRY_IND. */
946static st_index_t
947find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
948 st_data_t key, st_index_t *bin_ind)
949{
950 int eq_p, rebuilt_p;
951 st_index_t ind;
952 st_hash_t curr_hash_value = *hash_value;
953#ifdef QUADRATIC_PROBE
954 st_index_t d;
955#else
956 st_index_t peterb;
957#endif
958 st_index_t entry_index;
959 st_index_t first_deleted_bin_ind;
960 st_table_entry *entries;
961
962 ind = hash_bin(curr_hash_value, tab);
963#ifdef QUADRATIC_PROBE
964 d = 1;
965#else
966 peterb = curr_hash_value;
967#endif
968 FOUND_BIN;
969 first_deleted_bin_ind = UNDEFINED_BIN_IND;
970 entries = tab->entries;
971 for (;;) {
972 entry_index = get_bin(tab->bins, get_size_ind(tab), ind);
973 if (EMPTY_BIN_P(entry_index)) {
974 tab->num_entries++;
975 entry_index = UNDEFINED_ENTRY_IND;
976 if (first_deleted_bin_ind != UNDEFINED_BIN_IND) {
977 /* We can reuse bin of a deleted entry. */
978 ind = first_deleted_bin_ind;
979 MARK_BIN_EMPTY(tab, ind);
980 }
981 break;
982 }
983 else if (! DELETED_BIN_P(entry_index)) {
984 DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
985 if (EXPECT(rebuilt_p, 0))
986 return REBUILT_TABLE_ENTRY_IND;
987 if (eq_p)
988 break;
989 }
990 else if (first_deleted_bin_ind == UNDEFINED_BIN_IND)
991 first_deleted_bin_ind = ind;
992#ifdef QUADRATIC_PROBE
993 ind = hash_bin(ind + d, tab);
994 d++;
995#else
996 ind = secondary_hash(ind, tab, &peterb);
997#endif
998 COLLISION;
999 }
1000 *bin_ind = ind;
1001 return entry_index;
1002}
1003
1004/* Find an entry with KEY in table TAB. Return non-zero if we found
1005 it. Set up *RECORD to the found entry record. */
1006int
1007 st_lookup(st_table *tab, st_data_t key, st_data_t *value)
1008{
1009 st_index_t bin;
1010 st_hash_t hash = do_hash(key, tab);
1011
1012 retry:
1013 if (tab->bins == NULL) {
1014 bin = find_entry(tab, hash, key);
1015 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1016 goto retry;
1017 if (bin == UNDEFINED_ENTRY_IND)
1018 return 0;
1019 }
1020 else {
1021 bin = find_table_entry_ind(tab, hash, key);
1022 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1023 goto retry;
1024 if (bin == UNDEFINED_ENTRY_IND)
1025 return 0;
1026 bin -= ENTRY_BASE;
1027 }
1028 if (value != 0)
1029 *value = tab->entries[bin].record;
1030 return 1;
1031}
1032
1033/* Find an entry with KEY in table TAB. Return non-zero if we found
1034 it. Set up *RESULT to the found table entry key. */
1035int
1036 st_get_key(st_table *tab, st_data_t key, st_data_t *result)
1037{
1038 st_index_t bin;
1039 st_hash_t hash = do_hash(key, tab);
1040
1041 retry:
1042 if (tab->bins == NULL) {
1043 bin = find_entry(tab, hash, key);
1044 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1045 goto retry;
1046 if (bin == UNDEFINED_ENTRY_IND)
1047 return 0;
1048 }
1049 else {
1050 bin = find_table_entry_ind(tab, hash, key);
1051 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1052 goto retry;
1053 if (bin == UNDEFINED_ENTRY_IND)
1054 return 0;
1055 bin -= ENTRY_BASE;
1056 }
1057 if (result != 0)
1058 *result = tab->entries[bin].key;
1059 return 1;
1060}
1061
1062/* Check the table and rebuild it if it is necessary. */
1063static inline void
1064rebuild_table_if_necessary (st_table *tab)
1065{
1066 st_index_t bound = tab->entries_bound;
1067
1068 if (bound == get_allocated_entries(tab))
1069 rebuild_table(tab);
1070}
1071
1072/* Insert (KEY, VALUE) into table TAB and return zero. If there is
1073 already entry with KEY in the table, return nonzero and update
1074 the value of the found entry. */
1075int
1076 st_insert(st_table *tab, st_data_t key, st_data_t value)
1077{
1078 st_table_entry *entry;
1079 st_index_t bin;
1080 st_index_t ind;
1081 st_hash_t hash_value;
1082 st_index_t bin_ind;
1083 int new_p;
1084
1085 hash_value = do_hash(key, tab);
1086 retry:
1087 rebuild_table_if_necessary(tab);
1088 if (tab->bins == NULL) {
1089 bin = find_entry(tab, hash_value, key);
1090 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1091 goto retry;
1092 new_p = bin == UNDEFINED_ENTRY_IND;
1093 if (new_p)
1094 tab->num_entries++;
1095 bin_ind = UNDEFINED_BIN_IND;
1096 }
1097 else {
1098 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1099 key, &bin_ind);
1100 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1101 goto retry;
1102 new_p = bin == UNDEFINED_ENTRY_IND;
1103 bin -= ENTRY_BASE;
1104 }
1105 if (new_p) {
1106 ind = tab->entries_bound++;
1107 entry = &tab->entries[ind];
1108 entry->hash = hash_value;
1109 entry->key = key;
1110 entry->record = value;
1111 if (bin_ind != UNDEFINED_BIN_IND)
1112 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1113 return 0;
1114 }
1115 tab->entries[bin].record = value;
1116 return 1;
1117}
1118
1119/* Insert (KEY, VALUE, HASH) into table TAB. The table should not have
1120 entry with KEY before the insertion. */
1121static inline void
1122st_add_direct_with_hash(st_table *tab,
1123 st_data_t key, st_data_t value, st_hash_t hash)
1124{
1125 st_table_entry *entry;
1126 st_index_t ind;
1127 st_index_t bin_ind;
1128
1129 rebuild_table_if_necessary(tab);
1130 ind = tab->entries_bound++;
1131 entry = &tab->entries[ind];
1132 entry->hash = hash;
1133 entry->key = key;
1134 entry->record = value;
1135 tab->num_entries++;
1136 if (tab->bins != NULL) {
1137 bin_ind = find_table_bin_ind_direct(tab, hash, key);
1138 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1139 }
1140}
1141
1142/* Insert (KEY, VALUE) into table TAB. The table should not have
1143 entry with KEY before the insertion. */
1144void
1145 st_add_direct(st_table *tab, st_data_t key, st_data_t value)
1146{
1147 st_hash_t hash_value;
1148
1149 hash_value = do_hash(key, tab);
1150 st_add_direct_with_hash(tab, key, value, hash_value);
1151}
1152
1153/* Insert (FUNC(KEY), VALUE) into table TAB and return zero. If
1154 there is already entry with KEY in the table, return nonzero and
1155 update the value of the found entry. */
1156int
1157 st_insert2(st_table *tab, st_data_t key, st_data_t value,
1158 st_data_t (*func)(st_data_t))
1159{
1160 st_table_entry *entry;
1161 st_index_t bin;
1162 st_index_t ind;
1163 st_hash_t hash_value;
1164 st_index_t bin_ind;
1165 int new_p;
1166
1167 hash_value = do_hash(key, tab);
1168 retry:
1169 rebuild_table_if_necessary (tab);
1170 if (tab->bins == NULL) {
1171 bin = find_entry(tab, hash_value, key);
1172 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1173 goto retry;
1174 new_p = bin == UNDEFINED_ENTRY_IND;
1175 if (new_p)
1176 tab->num_entries++;
1177 bin_ind = UNDEFINED_BIN_IND;
1178 }
1179 else {
1180 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1181 key, &bin_ind);
1182 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1183 goto retry;
1184 new_p = bin == UNDEFINED_ENTRY_IND;
1185 bin -= ENTRY_BASE;
1186 }
1187 if (new_p) {
1188 key = (*func)(key);
1189 ind = tab->entries_bound++;
1190 entry = &tab->entries[ind];
1191 entry->hash = hash_value;
1192 entry->key = key;
1193 entry->record = value;
1194 if (bin_ind != UNDEFINED_BIN_IND)
1195 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1196 return 0;
1197 }
1198 tab->entries[bin].record = value;
1199 return 1;
1200}
1201
1202/* Create and return a copy of table OLD_TAB. */
1203 st_table *
1204 st_copy(st_table *old_tab)
1205{
1206 st_table *new_tab;
1207
1208 new_tab = (st_table *) malloc(sizeof(st_table));
1209#ifndef RUBY
1210 if (new_tab == NULL)
1211 return NULL;
1212#endif
1213 *new_tab = *old_tab;
1214 if (old_tab->bins == NULL)
1215 new_tab->bins = NULL;
1216 else {
1217 new_tab->bins = (st_index_t *) malloc(bins_size(old_tab));
1218#ifndef RUBY
1219 if (new_tab->bins == NULL) {
1220 free(new_tab);
1221 return NULL;
1222 }
1223#endif
1224 }
1225 new_tab->entries = (st_table_entry *) malloc(get_allocated_entries(old_tab)
1226 * sizeof(st_table_entry));
1227#ifndef RUBY
1228 if (new_tab->entries == NULL) {
1229 st_free_table(new_tab);
1230 return NULL;
1231 }
1232#endif
1233 MEMCPY(new_tab->entries, old_tab->entries, st_table_entry,
1234 get_allocated_entries(old_tab));
1235 if (old_tab->bins != NULL)
1236 MEMCPY(new_tab->bins, old_tab->bins, char, bins_size(old_tab));
1237 return new_tab;
1238}
1239
1240/* Update the entries start of table TAB after removing an entry
1241 with index N in the array entries. */
1242static inline void
1243update_range_for_deleted(st_table *tab, st_index_t n)
1244{
1245 /* Do not update entries_bound here. Otherwise, we can fill all
1246 bins by deleted entry value before rebuilding the table. */
1247 if (tab->entries_start == n)
1248 tab->entries_start = n + 1;
1249}
1250
1251/* Delete entry with KEY from table TAB, set up *VALUE (unless
1252 VALUE is zero) from deleted table entry, and return non-zero. If
1253 there is no entry with KEY in the table, clear *VALUE (unless VALUE
1254 is zero), and return zero. */
1255static int
1256st_general_delete(st_table *tab, st_data_t *key, st_data_t *value)
1257{
1258 st_table_entry *entry;
1259 st_index_t bin;
1260 st_index_t bin_ind;
1261 st_hash_t hash;
1262
1263 hash = do_hash(*key, tab);
1264 retry:
1265 if (tab->bins == NULL) {
1266 bin = find_entry(tab, hash, *key);
1267 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1268 goto retry;
1269 if (bin == UNDEFINED_ENTRY_IND) {
1270 if (value != 0) *value = 0;
1271 return 0;
1272 }
1273 }
1274 else {
1275 bin_ind = find_table_bin_ind(tab, hash, *key);
1276 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1277 goto retry;
1278 if (bin_ind == UNDEFINED_BIN_IND) {
1279 if (value != 0) *value = 0;
1280 return 0;
1281 }
1282 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1283 MARK_BIN_DELETED(tab, bin_ind);
1284 }
1285 entry = &tab->entries[bin];
1286 *key = entry->key;
1287 if (value != 0) *value = entry->record;
1288 MARK_ENTRY_DELETED(entry);
1289 tab->num_entries--;
1290 update_range_for_deleted(tab, bin);
1291 return 1;
1292}
1293
1294int
1295 st_delete(st_table *tab, st_data_t *key, st_data_t *value)
1296{
1297 return st_general_delete(tab, key, value);
1298}
1299
1300/* The function and other functions with suffix '_safe' or '_check'
1301 are originated from the previous implementation of the hash tables.
1302 It was necessary for correct deleting entries during traversing
1303 tables. The current implementation permits deletion during
1304 traversing without a specific way to do this. */
1305int
1306 st_delete_safe(st_table *tab, st_data_t *key, st_data_t *value,
1307 st_data_t never ATTRIBUTE_UNUSED)
1308{
1309 return st_general_delete(tab, key, value);
1310}
1311
1312/* If table TAB is empty, clear *VALUE (unless VALUE is zero), and
1313 return zero. Otherwise, remove the first entry in the table.
1314 Return its key through KEY and its record through VALUE (unless
1315 VALUE is zero). */
1316int
1317 st_shift(st_table *tab, st_data_t *key, st_data_t *value)
1318{
1319 st_index_t i, bound;
1320 st_index_t bin;
1321 st_table_entry *entries, *curr_entry_ptr;
1322 st_index_t bin_ind;
1323
1324 entries = tab->entries;
1325 bound = tab->entries_bound;
1326 for (i = tab->entries_start; i < bound; i++) {
1327 curr_entry_ptr = &entries[i];
1328 if (! DELETED_ENTRY_P(curr_entry_ptr)) {
1329 st_hash_t entry_hash = curr_entry_ptr->hash;
1330 st_data_t entry_key = curr_entry_ptr->key;
1331
1332 if (value != 0) *value = curr_entry_ptr->record;
1333 *key = entry_key;
1334 retry:
1335 if (tab->bins == NULL) {
1336 bin = find_entry(tab, entry_hash, entry_key);
1337 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0)) {
1338 entries = tab->entries;
1339 goto retry;
1340 }
1341 curr_entry_ptr = &entries[bin];
1342 }
1343 else {
1344 bin_ind = find_table_bin_ind(tab, entry_hash, entry_key);
1345 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0)) {
1346 entries = tab->entries;
1347 goto retry;
1348 }
1349 curr_entry_ptr = &entries[get_bin(tab->bins, get_size_ind(tab), bin_ind)
1350 - ENTRY_BASE];
1351 MARK_BIN_DELETED(tab, bin_ind);
1352 }
1353 MARK_ENTRY_DELETED(curr_entry_ptr);
1354 tab->num_entries--;
1355 update_range_for_deleted(tab, i);
1356 return 1;
1357 }
1358 }
1359 if (value != 0) *value = 0;
1360 return 0;
1361}
1362
1363/* See comments for function st_delete_safe. */
1364void
1365 st_cleanup_safe(st_table *tab ATTRIBUTE_UNUSED,
1366 st_data_t never ATTRIBUTE_UNUSED)
1367{
1368}
1369
1370/* Find entry with KEY in table TAB, call FUNC with pointers to copies
1371 of the key and the value of the found entry, and non-zero as the
1372 3rd argument. If the entry is not found, call FUNC with a pointer
1373 to KEY, a pointer to zero, and a zero argument. If the call
1374 returns ST_CONTINUE, the table will have an entry with key and
1375 value returned by FUNC through the 1st and 2nd parameters. If the
1376 call of FUNC returns ST_DELETE, the table will not have entry with
1377 KEY. The function returns flag of that the entry with KEY was in
1378 the table before the call. */
1379int
1380 st_update(st_table *tab, st_data_t key,
1381 st_update_callback_func *func, st_data_t arg)
1382{
1383 st_table_entry *entry = NULL; /* to avoid uninitialized value warning */
1384 st_index_t bin = 0; /* Ditto */
1385 st_table_entry *entries;
1386 st_index_t bin_ind;
1387 st_data_t value = 0, old_key;
1388 int retval, existing;
1389 st_hash_t hash = do_hash(key, tab);
1390
1391 retry:
1392 entries = tab->entries;
1393 if (tab->bins == NULL) {
1394 bin = find_entry(tab, hash, key);
1395 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1396 goto retry;
1397 existing = bin != UNDEFINED_ENTRY_IND;
1398 entry = &entries[bin];
1399 bin_ind = UNDEFINED_BIN_IND;
1400 }
1401 else {
1402 bin_ind = find_table_bin_ind(tab, hash, key);
1403 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1404 goto retry;
1405 existing = bin_ind != UNDEFINED_BIN_IND;
1406 if (existing) {
1407 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1408 entry = &entries[bin];
1409 }
1410 }
1411 if (existing) {
1412 key = entry->key;
1413 value = entry->record;
1414 }
1415 old_key = key;
1416 retval = (*func)(&key, &value, arg, existing);
1417 switch (retval) {
1418 case ST_CONTINUE:
1419 if (! existing) {
1420 st_add_direct_with_hash(tab, key, value, hash);
1421 break;
1422 }
1423 if (old_key != key) {
1424 entry->key = key;
1425 }
1426 entry->record = value;
1427 break;
1428 case ST_DELETE:
1429 if (existing) {
1430 if (bin_ind != UNDEFINED_BIN_IND)
1431 MARK_BIN_DELETED(tab, bin_ind);
1432 MARK_ENTRY_DELETED(entry);
1433 tab->num_entries--;
1434 update_range_for_deleted(tab, bin);
1435 }
1436 break;
1437 }
1438 return existing;
1439}
1440
1441/* Traverse all entries in table TAB calling FUNC with current entry
1442 key and value and zero. If the call returns ST_STOP, stop
1443 traversing. If the call returns ST_DELETE, delete the current
1444 entry from the table. In case of ST_CHECK or ST_CONTINUE, continue
1445 traversing. The function returns zero unless an error is found.
1446 CHECK_P is flag of st_foreach_check call. The behavior is a bit
1447 different for ST_CHECK and when the current element is removed
1448 during traversing. */
1449static inline int
1450st_general_foreach(st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg,
1451 int check_p)
1452{
1453 st_index_t bin;
1454 st_index_t bin_ind;
1455 st_table_entry *entries, *curr_entry_ptr;
1456 enum st_retval retval;
1457 st_index_t i, rebuilds_num;
1458 st_hash_t hash;
1459 st_data_t key;
1460 int error_p, packed_p = tab->bins == NULL;
1461
1462 entries = tab->entries;
1463 /* The bound can change inside the loop even without rebuilding
1464 the table, e.g. by an entry insertion. */
1465 for (i = tab->entries_start; i < tab->entries_bound; i++) {
1466 curr_entry_ptr = &entries[i];
1467 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
1468 continue;
1469 key = curr_entry_ptr->key;
1470 rebuilds_num = tab->rebuilds_num;
1471 hash = curr_entry_ptr->hash;
1472 retval = (*func)(key, curr_entry_ptr->record, arg, 0);
1473
1474 if (retval == ST_REPLACE && replace) {
1475 st_data_t value;
1476 value = curr_entry_ptr->record;
1477 retval = (*replace)(&key, &value, arg, TRUE);
1478 curr_entry_ptr->key = key;
1479 curr_entry_ptr->record = value;
1480 }
1481
1482 if (rebuilds_num != tab->rebuilds_num) {
1483 retry:
1484 entries = tab->entries;
1485 packed_p = tab->bins == NULL;
1486 if (packed_p) {
1487 i = find_entry(tab, hash, key);
1488 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1489 goto retry;
1490 error_p = i == UNDEFINED_ENTRY_IND;
1491 }
1492 else {
1493 i = find_table_entry_ind(tab, hash, key);
1494 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1495 goto retry;
1496 error_p = i == UNDEFINED_ENTRY_IND;
1497 i -= ENTRY_BASE;
1498 }
1499 if (error_p && check_p) {
1500 /* call func with error notice */
1501 retval = (*func)(0, 0, arg, 1);
1502 return 1;
1503 }
1504 curr_entry_ptr = &entries[i];
1505 }
1506 switch (retval) {
1507 case ST_REPLACE:
1508 break;
1509 case ST_CONTINUE:
1510 break;
1511 case ST_CHECK:
1512 if (check_p)
1513 break;
1514 case ST_STOP:
1515 return 0;
1516 case ST_DELETE: {
1517 st_data_t key = curr_entry_ptr->key;
1518
1519 again:
1520 if (packed_p) {
1521 bin = find_entry(tab, hash, key);
1522 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1523 goto again;
1524 if (bin == UNDEFINED_ENTRY_IND)
1525 break;
1526 }
1527 else {
1528 bin_ind = find_table_bin_ind(tab, hash, key);
1529 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1530 goto again;
1531 if (bin_ind == UNDEFINED_BIN_IND)
1532 break;
1533 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1534 MARK_BIN_DELETED(tab, bin_ind);
1535 }
1536 curr_entry_ptr = &entries[bin];
1537 MARK_ENTRY_DELETED(curr_entry_ptr);
1538 tab->num_entries--;
1539 update_range_for_deleted(tab, bin);
1540 break;
1541 }
1542 }
1543 }
1544 return 0;
1545}
1546
1547int
1548 st_foreach_with_replace(st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg)
1549{
1550 return st_general_foreach(tab, func, replace, arg, TRUE);
1551}
1552
1553struct functor {
1554 st_foreach_callback_func *func;
1555 st_data_t arg;
1556};
1557
1558static int
1559apply_functor(st_data_t k, st_data_t v, st_data_t d, int _)
1560{
1561 const struct functor *f = (void *)d;
1562 return f->func(k, v, f->arg);
1563}
1564
1565int
1566 st_foreach(st_table *tab, st_foreach_callback_func *func, st_data_t arg)
1567{
1568 const struct functor f = { func, arg };
1569 return st_general_foreach(tab, apply_functor, 0, (st_data_t)&f, FALSE);
1570}
1571
1572/* See comments for function st_delete_safe. */
1573int
1574 st_foreach_check(st_table *tab, st_foreach_check_callback_func *func, st_data_t arg,
1575 st_data_t never ATTRIBUTE_UNUSED)
1576{
1577 return st_general_foreach(tab, func, 0, arg, TRUE);
1578}
1579
1580/* Set up array KEYS by at most SIZE keys of head table TAB entries.
1581 Return the number of keys set up in array KEYS. */
1582static inline st_index_t
1583st_general_keys(st_table *tab, st_data_t *keys, st_index_t size)
1584{
1585 st_index_t i, bound;
1586 st_data_t key, *keys_start, *keys_end;
1587 st_table_entry *curr_entry_ptr, *entries = tab->entries;
1588
1589 bound = tab->entries_bound;
1590 keys_start = keys;
1591 keys_end = keys + size;
1592 for (i = tab->entries_start; i < bound; i++) {
1593 if (keys == keys_end)
1594 break;
1595 curr_entry_ptr = &entries[i];
1596 key = curr_entry_ptr->key;
1597 if (! DELETED_ENTRY_P(curr_entry_ptr))
1598 *keys++ = key;
1599 }
1600
1601 return keys - keys_start;
1602}
1603
1604 st_index_t
1605 st_keys(st_table *tab, st_data_t *keys, st_index_t size)
1606{
1607 return st_general_keys(tab, keys, size);
1608}
1609
1610/* See comments for function st_delete_safe. */
1611 st_index_t
1612 st_keys_check(st_table *tab, st_data_t *keys, st_index_t size,
1613 st_data_t never ATTRIBUTE_UNUSED)
1614{
1615 return st_general_keys(tab, keys, size);
1616}
1617
1618/* Set up array VALUES by at most SIZE values of head table TAB
1619 entries. Return the number of values set up in array VALUES. */
1620static inline st_index_t
1621st_general_values(st_table *tab, st_data_t *values, st_index_t size)
1622{
1623 st_index_t i, bound;
1624 st_data_t *values_start, *values_end;
1625 st_table_entry *curr_entry_ptr, *entries = tab->entries;
1626
1627 values_start = values;
1628 values_end = values + size;
1629 bound = tab->entries_bound;
1630 for (i = tab->entries_start; i < bound; i++) {
1631 if (values == values_end)
1632 break;
1633 curr_entry_ptr = &entries[i];
1634 if (! DELETED_ENTRY_P(curr_entry_ptr))
1635 *values++ = curr_entry_ptr->record;
1636 }
1637
1638 return values - values_start;
1639}
1640
1641 st_index_t
1642 st_values(st_table *tab, st_data_t *values, st_index_t size)
1643{
1644 return st_general_values(tab, values, size);
1645}
1646
1647/* See comments for function st_delete_safe. */
1648 st_index_t
1649 st_values_check(st_table *tab, st_data_t *values, st_index_t size,
1650 st_data_t never ATTRIBUTE_UNUSED)
1651{
1652 return st_general_values(tab, values, size);
1653}
1654
1655#define FNV1_32A_INIT 0x811c9dc5
1656
1657/*
1658 * 32 bit magic FNV-1a prime
1659 */
1660#define FNV_32_PRIME 0x01000193
1661
1662#ifndef UNALIGNED_WORD_ACCESS
1663# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \
1664 defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || \
1665 defined(__powerpc64__) || defined(__aarch64__) || \
1666 defined(__mc68020__)
1667# define UNALIGNED_WORD_ACCESS 1
1668# endif
1669#endif
1670#ifndef UNALIGNED_WORD_ACCESS
1671# define UNALIGNED_WORD_ACCESS 0
1672#endif
1673
1674/* This hash function is quite simplified MurmurHash3
1675 * Simplification is legal, cause most of magic still happens in finalizator.
1676 * And finalizator is almost the same as in MurmurHash3 */
1677#define BIG_CONSTANT(x,y) ((st_index_t)(x)<<32|(st_index_t)(y))
1678#define ROTL(x,n) ((x)<<(n)|(x)>>(SIZEOF_ST_INDEX_T*CHAR_BIT-(n)))
1679
1680#if ST_INDEX_BITS <= 32
1681#define C1 (st_index_t)0xcc9e2d51
1682#define C2 (st_index_t)0x1b873593
1683#else
1684#define C1 BIG_CONSTANT(0x87c37b91,0x114253d5);
1685#define C2 BIG_CONSTANT(0x4cf5ad43,0x2745937f);
1686#endif
1687 NO_SANITIZE("unsigned-integer-overflow", static inline st_index_t murmur_step(st_index_t h, st_index_t k));
1688 NO_SANITIZE("unsigned-integer-overflow", static inline st_index_t murmur_finish(st_index_t h));
1689 NO_SANITIZE("unsigned-integer-overflow", extern st_index_t st_hash(const void *ptr, size_t len, st_index_t h));
1690
1691static inline st_index_t
1692murmur_step(st_index_t h, st_index_t k)
1693{
1694#if ST_INDEX_BITS <= 32
1695#define r1 (17)
1696#define r2 (11)
1697#else
1698#define r1 (33)
1699#define r2 (24)
1700#endif
1701 k *= C1;
1702 h ^= ROTL(k, r1);
1703 h *= C2;
1704 h = ROTL(h, r2);
1705 return h;
1706}
1707#undef r1
1708#undef r2
1709
1710static inline st_index_t
1711murmur_finish(st_index_t h)
1712{
1713#if ST_INDEX_BITS <= 32
1714#define r1 (16)
1715#define r2 (13)
1716#define r3 (16)
1717 const st_index_t c1 = 0x85ebca6b;
1718 const st_index_t c2 = 0xc2b2ae35;
1719#else
1720/* values are taken from Mix13 on http://zimbry.blogspot.ru/2011/09/better-bit-mixing-improving-on.html */
1721#define r1 (30)
1722#define r2 (27)
1723#define r3 (31)
1724 const st_index_t c1 = BIG_CONSTANT(0xbf58476d,0x1ce4e5b9);
1725 const st_index_t c2 = BIG_CONSTANT(0x94d049bb,0x133111eb);
1726#endif
1727#if ST_INDEX_BITS > 64
1728 h ^= h >> 64;
1729 h *= c2;
1730 h ^= h >> 65;
1731#endif
1732 h ^= h >> r1;
1733 h *= c1;
1734 h ^= h >> r2;
1735 h *= c2;
1736 h ^= h >> r3;
1737 return h;
1738}
1739#undef r1
1740#undef r2
1741#undef r3
1742
1743 st_index_t
1744 st_hash(const void *ptr, size_t len, st_index_t h)
1745{
1746 const char *data = ptr;
1747 st_index_t t = 0;
1748 size_t l = len;
1749
1750#define data_at(n) (st_index_t)((unsigned char)data[(n)])
1751#define UNALIGNED_ADD_4 UNALIGNED_ADD(2); UNALIGNED_ADD(1); UNALIGNED_ADD(0)
1752#if SIZEOF_ST_INDEX_T > 4
1753#define UNALIGNED_ADD_8 UNALIGNED_ADD(6); UNALIGNED_ADD(5); UNALIGNED_ADD(4); UNALIGNED_ADD(3); UNALIGNED_ADD_4
1754#if SIZEOF_ST_INDEX_T > 8
1755#define UNALIGNED_ADD_16 UNALIGNED_ADD(14); UNALIGNED_ADD(13); UNALIGNED_ADD(12); UNALIGNED_ADD(11); \
1756 UNALIGNED_ADD(10); UNALIGNED_ADD(9); UNALIGNED_ADD(8); UNALIGNED_ADD(7); UNALIGNED_ADD_8
1757#define UNALIGNED_ADD_ALL UNALIGNED_ADD_16
1758#endif
1759#define UNALIGNED_ADD_ALL UNALIGNED_ADD_8
1760#else
1761#define UNALIGNED_ADD_ALL UNALIGNED_ADD_4
1762#endif
1763#undef SKIP_TAIL
1764 if (len >= sizeof(st_index_t)) {
1765#if !UNALIGNED_WORD_ACCESS
1766 int align = (int)((st_data_t)data % sizeof(st_index_t));
1767 if (align) {
1768 st_index_t d = 0;
1769 int sl, sr, pack;
1770
1771 switch (align) {
1772#ifdef WORDS_BIGENDIAN
1773# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1774 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 2)
1775#else
1776# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1777 t |= data_at(n) << CHAR_BIT*(n)
1778#endif
1779 UNALIGNED_ADD_ALL;
1780#undef UNALIGNED_ADD
1781 }
1782
1783#ifdef WORDS_BIGENDIAN
1784 t >>= (CHAR_BIT * align) - CHAR_BIT;
1785#else
1786 t <<= (CHAR_BIT * align);
1787#endif
1788
1789 data += sizeof(st_index_t)-align;
1790 len -= sizeof(st_index_t)-align;
1791
1792 sl = CHAR_BIT * (SIZEOF_ST_INDEX_T-align);
1793 sr = CHAR_BIT * align;
1794
1795 while (len >= sizeof(st_index_t)) {
1796 d = *(st_index_t *)data;
1797#ifdef WORDS_BIGENDIAN
1798 t = (t << sr) | (d >> sl);
1799#else
1800 t = (t >> sr) | (d << sl);
1801#endif
1802 h = murmur_step(h, t);
1803 t = d;
1804 data += sizeof(st_index_t);
1805 len -= sizeof(st_index_t);
1806 }
1807
1808 pack = len < (size_t)align ? (int)len : align;
1809 d = 0;
1810 switch (pack) {
1811#ifdef WORDS_BIGENDIAN
1812# define UNALIGNED_ADD(n) case (n) + 1: \
1813 d |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1814#else
1815# define UNALIGNED_ADD(n) case (n) + 1: \
1816 d |= data_at(n) << CHAR_BIT*(n)
1817#endif
1818 UNALIGNED_ADD_ALL;
1819#undef UNALIGNED_ADD
1820 }
1821#ifdef WORDS_BIGENDIAN
1822 t = (t << sr) | (d >> sl);
1823#else
1824 t = (t >> sr) | (d << sl);
1825#endif
1826
1827 if (len < (size_t)align) goto skip_tail;
1828# define SKIP_TAIL 1
1829 h = murmur_step(h, t);
1830 data += pack;
1831 len -= pack;
1832 }
1833 else
1834#endif
1835#ifdef HAVE_BUILTIN___BUILTIN_ASSUME_ALIGNED
1836#define aligned_data __builtin_assume_aligned(data, sizeof(st_index_t))
1837#else
1838#define aligned_data data
1839#endif
1840 {
1841 do {
1842 h = murmur_step(h, *(st_index_t *)aligned_data);
1843 data += sizeof(st_index_t);
1844 len -= sizeof(st_index_t);
1845 } while (len >= sizeof(st_index_t));
1846 }
1847 }
1848
1849 t = 0;
1850 switch (len) {
1851#if UNALIGNED_WORD_ACCESS && SIZEOF_ST_INDEX_T <= 8 && CHAR_BIT == 8
1852 /* in this case byteorder doesn't really matter */
1853#if SIZEOF_ST_INDEX_T > 4
1854 case 7: t |= data_at(6) << 48;
1855 case 6: t |= data_at(5) << 40;
1856 case 5: t |= data_at(4) << 32;
1857 case 4:
1858 t |= (st_index_t)*(uint32_t*)aligned_data;
1859 goto skip_tail;
1860# define SKIP_TAIL 1
1861#endif
1862 case 3: t |= data_at(2) << 16;
1863 case 2: t |= data_at(1) << 8;
1864 case 1: t |= data_at(0);
1865#else
1866#ifdef WORDS_BIGENDIAN
1867# define UNALIGNED_ADD(n) case (n) + 1: \
1868 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1869#else
1870# define UNALIGNED_ADD(n) case (n) + 1: \
1871 t |= data_at(n) << CHAR_BIT*(n)
1872#endif
1873 UNALIGNED_ADD_ALL;
1874#undef UNALIGNED_ADD
1875#endif
1876#ifdef SKIP_TAIL
1877 skip_tail:
1878#endif
1879 h ^= t; h -= ROTL(t, 7);
1880 h *= C2;
1881 }
1882 h ^= l;
1883#undef aligned_data
1884
1885 return murmur_finish(h);
1886}
1887
1888 st_index_t
1889 st_hash_uint32(st_index_t h, uint32_t i)
1890{
1891 return murmur_step(h, i);
1892}
1893
1894 NO_SANITIZE("unsigned-integer-overflow", extern st_index_t st_hash_uint(st_index_t h, st_index_t i));
1895 st_index_t
1896 st_hash_uint(st_index_t h, st_index_t i)
1897{
1898 i += h;
1899/* no matter if it is BigEndian or LittleEndian,
1900 * we hash just integers */
1901#if SIZEOF_ST_INDEX_T*CHAR_BIT > 8*8
1902 h = murmur_step(h, i >> 8*8);
1903#endif
1904 h = murmur_step(h, i);
1905 return h;
1906}
1907
1908 st_index_t
1909 st_hash_end(st_index_t h)
1910{
1911 h = murmur_finish(h);
1912 return h;
1913}
1914
1915#undef st_hash_start
1916 st_index_t
1917 rb_st_hash_start(st_index_t h)
1918{
1919 return h;
1920}
1921
1922static st_index_t
1923strhash(st_data_t arg)
1924{
1925 register const char *string = (const char *)arg;
1926 return st_hash(string, strlen(string), FNV1_32A_INIT);
1927}
1928
1929int
1930 st_locale_insensitive_strcasecmp(const char *s1, const char *s2)
1931{
1932 char c1, c2;
1933
1934 while (1) {
1935 c1 = *s1++;
1936 c2 = *s2++;
1937 if (c1 == '\0' || c2 == '\0') {
1938 if (c1 != '\0') return 1;
1939 if (c2 != '\0') return -1;
1940 return 0;
1941 }
1942 if (('A' <= c1) && (c1 <= 'Z')) c1 += 'a' - 'A';
1943 if (('A' <= c2) && (c2 <= 'Z')) c2 += 'a' - 'A';
1944 if (c1 != c2) {
1945 if (c1 > c2)
1946 return 1;
1947 else
1948 return -1;
1949 }
1950 }
1951}
1952
1953int
1954 st_locale_insensitive_strncasecmp(const char *s1, const char *s2, size_t n)
1955{
1956 char c1, c2;
1957 size_t i;
1958
1959 for (i = 0; i < n; i++) {
1960 c1 = *s1++;
1961 c2 = *s2++;
1962 if (c1 == '\0' || c2 == '\0') {
1963 if (c1 != '\0') return 1;
1964 if (c2 != '\0') return -1;
1965 return 0;
1966 }
1967 if (('A' <= c1) && (c1 <= 'Z')) c1 += 'a' - 'A';
1968 if (('A' <= c2) && (c2 <= 'Z')) c2 += 'a' - 'A';
1969 if (c1 != c2) {
1970 if (c1 > c2)
1971 return 1;
1972 else
1973 return -1;
1974 }
1975 }
1976 return 0;
1977}
1978
1979static int
1980st_strcmp(st_data_t lhs, st_data_t rhs)
1981{
1982 const char *s1 = (char *)lhs;
1983 const char *s2 = (char *)rhs;
1984 return strcmp(s1, s2);
1985}
1986
1987static int
1988st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs)
1989{
1990 const char *s1 = (char *)lhs;
1991 const char *s2 = (char *)rhs;
1992 return st_locale_insensitive_strcasecmp(s1, s2);
1993}
1994
1995 NO_SANITIZE("unsigned-integer-overflow", PUREFUNC(static st_index_t strcasehash(st_data_t)));
1996static st_index_t
1997strcasehash(st_data_t arg)
1998{
1999 register const char *string = (const char *)arg;
2000 register st_index_t hval = FNV1_32A_INIT;
2001
2002 /*
2003 * FNV-1a hash each octet in the buffer
2004 */
2005 while (*string) {
2006 unsigned int c = (unsigned char)*string++;
2007 if ((unsigned int)(c - 'A') <= ('Z' - 'A')) c += 'a' - 'A';
2008 hval ^= c;
2009
2010 /* multiply by the 32 bit FNV magic prime mod 2^32 */
2011 hval *= FNV_32_PRIME;
2012 }
2013 return hval;
2014}
2015
2016int
2017 st_numcmp(st_data_t x, st_data_t y)
2018{
2019 return x != y;
2020}
2021
2022 st_index_t
2023 st_numhash(st_data_t n)
2024{
2025 enum {s1 = 11, s2 = 3};
2026 return (st_index_t)((n>>s1|(n<<s2)) ^ (n>>s2));
2027}
2028
2029/* Expand TAB to be suitable for holding SIZ entries in total.
2030 Pre-existing entries remain not deleted inside of TAB, but its bins
2031 are cleared to expect future reconstruction. See rehash below. */
2032static void
2033st_expand_table(st_table *tab, st_index_t siz)
2034{
2035 st_table *tmp;
2036 st_index_t n;
2037
2038 if (siz <= get_allocated_entries(tab))
2039 return; /* enough room already */
2040
2041 tmp = st_init_table_with_size(tab->type, siz);
2042 n = get_allocated_entries(tab);
2043 MEMCPY(tmp->entries, tab->entries, st_table_entry, n);
2044 free(tab->entries);
2045 if (tab->bins != NULL)
2046 free(tab->bins);
2047 if (tmp->bins != NULL)
2048 free(tmp->bins);
2049 tab->entry_power = tmp->entry_power;
2050 tab->bin_power = tmp->bin_power;
2051 tab->size_ind = tmp->size_ind;
2052 tab->entries = tmp->entries;
2053 tab->bins = NULL;
2054 tab->rebuilds_num++;
2055 free(tmp);
2056}
2057
2058/* Rehash using linear search. Return TRUE if we found that the table
2059 was rebuilt. */
2060static int
2061st_rehash_linear(st_table *tab)
2062{
2063 int eq_p, rebuilt_p;
2064 st_index_t i, j;
2065 st_table_entry *p, *q;
2066 if (tab->bins) {
2067 free(tab->bins);
2068 tab->bins = NULL;
2069 }
2070 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2071 p = &tab->entries[i];
2072 if (DELETED_ENTRY_P(p))
2073 continue;
2074 for (j = i + 1; j < tab->entries_bound; j++) {
2075 q = &tab->entries[j];
2076 if (DELETED_ENTRY_P(q))
2077 continue;
2078 DO_PTR_EQUAL_CHECK(tab, p, q->hash, q->key, eq_p, rebuilt_p);
2079 if (EXPECT(rebuilt_p, 0))
2080 return TRUE;
2081 if (eq_p) {
2082 *p = *q;
2083 MARK_ENTRY_DELETED(q);
2084 tab->num_entries--;
2085 update_range_for_deleted(tab, j);
2086 }
2087 }
2088 }
2089 return FALSE;
2090}
2091
2092/* Rehash using index. Return TRUE if we found that the table was
2093 rebuilt. */
2094static int
2095st_rehash_indexed(st_table *tab)
2096{
2097 int eq_p, rebuilt_p;
2098 st_index_t i;
2099 st_index_t const n = bins_size(tab);
2100 unsigned int const size_ind = get_size_ind(tab);
2101 st_index_t *bins = realloc(tab->bins, n);
2102 tab->bins = bins;
2103 initialize_bins(tab);
2104 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2105 st_table_entry *p = &tab->entries[i];
2106 st_index_t ind;
2107#ifdef QUADRATIC_PROBE
2108 st_index_t d = 1;
2109#else
2110 st_index_t peterb = p->hash;
2111#endif
2112
2113 if (DELETED_ENTRY_P(p))
2114 continue;
2115
2116 ind = hash_bin(p->hash, tab);
2117 for(;;) {
2118 st_index_t bin = get_bin(bins, size_ind, ind);
2119 if (EMPTY_OR_DELETED_BIN_P(bin)) {
2120 /* ok, new room */
2121 set_bin(bins, size_ind, ind, i + ENTRY_BASE);
2122 break;
2123 }
2124 else {
2125 st_table_entry *q = &tab->entries[bin - ENTRY_BASE];
2126 DO_PTR_EQUAL_CHECK(tab, q, p->hash, p->key, eq_p, rebuilt_p);
2127 if (EXPECT(rebuilt_p, 0))
2128 return TRUE;
2129 if (eq_p) {
2130 /* duplicated key; delete it */
2131 q->record = p->record;
2132 MARK_ENTRY_DELETED(p);
2133 tab->num_entries--;
2134 update_range_for_deleted(tab, bin);
2135 break;
2136 }
2137 else {
2138 /* hash collision; skip it */
2139#ifdef QUADRATIC_PROBE
2140 ind = hash_bin(ind + d, tab);
2141 d++;
2142#else
2143 ind = secondary_hash(ind, tab, &peterb);
2144#endif
2145 }
2146 }
2147 }
2148 }
2149 return FALSE;
2150}
2151
2152/* Reconstruct TAB's bins according to TAB's entries. This function
2153 permits conflicting keys inside of entries. No errors are reported
2154 then. All but one of them are discarded silently. */
2155static void
2156st_rehash(st_table *tab)
2157{
2158 int rebuilt_p;
2159
2160 do {
2161 if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2162 rebuilt_p = st_rehash_linear(tab);
2163 else
2164 rebuilt_p = st_rehash_indexed(tab);
2165 } while (rebuilt_p);
2166}
2167
2168#ifdef RUBY
2169static st_data_t
2170st_stringify(VALUE key)
2171{
2172 return (rb_obj_class(key) == rb_cString && !RB_OBJ_FROZEN(key)) ?
2173 rb_hash_key_str(key) : key;
2174}
2175
2176static void
2177st_insert_single(st_table *tab, VALUE hash, VALUE key, VALUE val)
2178{
2179 st_data_t k = st_stringify(key);
2180 st_table_entry e;
2181 e.hash = do_hash(k, tab);
2182 e.key = k;
2183 e.record = val;
2184
2185 tab->entries[tab->entries_bound++] = e;
2186 tab->num_entries++;
2187 RB_OBJ_WRITTEN(hash, Qundef, k);
2188 RB_OBJ_WRITTEN(hash, Qundef, val);
2189}
2190
2191static void
2192st_insert_linear(st_table *tab, long argc, const VALUE *argv, VALUE hash)
2193{
2194 long i;
2195
2196 for (i = 0; i < argc; /* */) {
2197 st_data_t k = st_stringify(argv[i++]);
2198 st_data_t v = argv[i++];
2199 st_insert(tab, k, v);
2200 RB_OBJ_WRITTEN(hash, Qundef, k);
2201 RB_OBJ_WRITTEN(hash, Qundef, v);
2202 }
2203}
2204
2205static void
2206st_insert_generic(st_table *tab, long argc, const VALUE *argv, VALUE hash)
2207{
2208 long i;
2209
2210 /* push elems */
2211 for (i = 0; i < argc; /* */) {
2212 VALUE key = argv[i++];
2213 VALUE val = argv[i++];
2214 st_insert_single(tab, hash, key, val);
2215 }
2216
2217 /* reindex */
2218 st_rehash(tab);
2219}
2220
2221/* Mimics ruby's { foo => bar } syntax. This function is subpart
2222 of rb_hash_bulk_insert. */
2223void
2224 rb_hash_bulk_insert_into_st_table(long argc, const VALUE *argv, VALUE hash)
2225{
2226 st_index_t n, size = argc / 2;
2227 st_table *tab = RHASH_ST_TABLE(hash);
2228
2229 tab = RHASH_TBL_RAW(hash);
2230 n = tab->entries_bound + size;
2231 st_expand_table(tab, n);
2232 if (UNLIKELY(tab->num_entries))
2233 st_insert_generic(tab, argc, argv, hash);
2234 else if (argc <= 2)
2235 st_insert_single(tab, hash, argv[0], argv[1]);
2236 else if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2237 st_insert_linear(tab, argc, argv, hash);
2238 else
2239 st_insert_generic(tab, argc, argv, hash);
2240}
2241
2242// to iterate iv_index_tbl
2243 st_data_t
2244rb_st_nth_key(st_table *tab, st_index_t index)
2245{
2246 if (LIKELY(tab->entries_start == 0 &&
2247 tab->num_entries == tab->entries_bound &&
2248 index < tab->num_entries)) {
2249 return tab->entries[index].key;
2250 }
2251 else {
2252 rb_bug("unreachable");
2253 }
2254}
2255
2256#endif
assert.h
PUREFUNC
#define PUREFUNC(x)
Definition: attributes.h:54
bits.h
Internal header for bitwise integer algorithms.
ptr
struct RIMemo * ptr
Definition: debug.c:88
num
big_t * num
Definition: enough.c:232
len
uint8_t len
Definition: escape.c:17
str
char str[HTML_ESCAPE_MAX_LEN+1]
Definition: escape.c:18
UNLIKELY
#define UNLIKELY(x)
Definition: ffi_common.h:126
LIKELY
#define LIKELY(x)
Definition: ffi_common.h:125
rb_cString
VALUE rb_cString
Definition: string.c:80
rb_raise
void rb_raise(VALUE exc, const char *fmt,...)
Definition: error.c:2917
rb_bug
void rb_bug(const char *fmt,...)
Definition: error.c:768
rb_eRuntimeError
VALUE rb_eRuntimeError
Definition: error.c:1055
rb_obj_class
VALUE rb_obj_class(VALUE)
Definition: object.c:245
rb_hash_key_str
VALUE rb_hash_key_str(VALUE key)
Definition: hash.c:2852
st_hash_t
st_index_t st_hash_t
Definition: hash.c:341
hash.h
Internal header for Hash.
RHASH_TBL_RAW
#define RHASH_TBL_RAW(h)
Definition: hash.h:118
internal.h
size
voidpf void uLong size
Definition: ioapi.h:138
long
typedef long(ZCALLBACK *tell_file_func) OF((voidpf opaque
int
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
CHAR_BIT
#define CHAR_BIT
Definition: limits.h:44
MEMCPY
#define MEMCPY(p1, p2, type, n)
Definition: memory.h:129
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition: cxxanyargs.hpp:56
TRUE
#define TRUE
Definition: nkf.h:175
FALSE
#define FALSE
Definition: nkf.h:174
key
key
Definition: openssl_missing.h:185
NULL
#define NULL
Definition: regenc.h:69
regint.h
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(a, oldv, b)
WB for new reference from ‘a’ to ‘b’.
Definition: rgengc.h:114
argc
int argc
Definition: ruby.c:240
argv
char ** argv
Definition: ruby.c:241
sanitizers.h
Internal header for ASAN / MSAN / etc.
NO_SANITIZE
#define NO_SANITIZE(x, y)
Definition: sanitizers.h:61
uint32_t
unsigned int uint32_t
Definition: sha2.h:101
Qundef
#define Qundef
Definition: special_consts.h:53
f
#define f
RESERVED_HASH_SUBSTITUTION_VAL
#define RESERVED_HASH_SUBSTITUTION_VAL
Definition: st.c:314
UNDEFINED_BIN_IND
#define UNDEFINED_BIN_IND
Definition: st.c:390
r2
#define r2
type_numhash
#define type_numhash
Definition: st.c:139
realloc
#define realloc
Definition: st.c:172
MAX_POWER2_FOR_TABLES_WITHOUT_BINS
#define MAX_POWER2_FOR_TABLES_WITHOUT_BINS
Definition: st.c:336
ENTRY_BASE
#define ENTRY_BASE
Definition: st.c:381
MARK_BIN_DELETED
#define MARK_BIN_DELETED(tab, i)
Definition: st.c:400
MINIMAL_POWER2
#define MINIMAL_POWER2
Definition: st.c:328
free
#define free
Definition: st.c:173
MAX_POWER2
#define MAX_POWER2
Definition: st.c:274
FNV1_32A_INIT
#define FNV1_32A_INIT
Definition: st.c:1655
C1
#define C1
Definition: st.c:1681
MARK_BIN_EMPTY
#define MARK_BIN_EMPTY(tab, i)
Definition: st.c:385
REBUILT_TABLE_BIN_IND
#define REBUILT_TABLE_BIN_IND
Definition: st.c:395
DELETED_ENTRY_P
#define DELETED_ENTRY_P(e_ptr)
Definition: st.c:420
EMPTY_BIN_P
#define EMPTY_BIN_P(b)
Definition: st.c:407
rb_st_hash_start
st_index_t rb_st_hash_start(st_index_t h)
Definition: st.c:1917
data_at
#define data_at(n)
UNALIGNED_ADD_ALL
#define UNALIGNED_ADD_ALL
FOUND_BIN
#define FOUND_BIN
Definition: st.c:687
REBUILD_THRESHOLD
#define REBUILD_THRESHOLD
Definition: st.c:693
ATTRIBUTE_UNUSED
#define ATTRIBUTE_UNUSED
Definition: st.c:123
st_init_numtable
st_table * st_init_numtable(void)
Definition: st.c:583
st_init_strtable
st_table * st_init_strtable(void)
Definition: st.c:598
UNDEFINED_ENTRY_IND
#define UNDEFINED_ENTRY_IND
Definition: st.c:389
r1
#define r1
aligned_data
#define aligned_data
ROTL
#define ROTL(x, n)
Definition: st.c:1678
rb_hash_bulk_insert_into_st_table
void rb_hash_bulk_insert_into_st_table(long argc, const VALUE *argv, VALUE hash)
Definition: st.c:2224
DO_PTR_EQUAL_CHECK
#define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p)
Definition: st.c:182
RESERVED_HASH_VAL
#define RESERVED_HASH_VAL
Definition: st.c:313
COLLISION
#define COLLISION
Definition: st.c:686
EXPECT
#define EXPECT(expr, val)
Definition: st.c:122
rb_st_nth_key
st_data_t rb_st_nth_key(st_table *tab, st_index_t index)
Definition: st.c:2244
EMPTY_OR_DELETED_BIN_P
#define EMPTY_OR_DELETED_BIN_P(b)
Definition: st.c:409
REBUILT_TABLE_ENTRY_IND
#define REBUILT_TABLE_ENTRY_IND
Definition: st.c:394
st_hash_t
st_index_t st_hash_t
Definition: st.c:131
BIG_CONSTANT
#define BIG_CONSTANT(x, y)
Definition: st.c:1677
malloc
#define malloc
Definition: st.c:170
C2
#define C2
Definition: st.c:1682
PREFETCH
#define PREFETCH(addr, write_p)
Definition: st.c:121
FNV_32_PRIME
#define FNV_32_PRIME
Definition: st.c:1660
DELETED_BIN_P
#define DELETED_BIN_P(b)
Definition: st.c:408
r3
#define r3
st_init_strcasetable
st_table * st_init_strcasetable(void)
Definition: st.c:613
MARK_ENTRY_DELETED
#define MARK_ENTRY_DELETED(e_ptr)
Definition: st.c:419
st.h
st_delete_safe
#define st_delete_safe
Definition: st.h:120
st_retval
st_retval
Definition: st.h:99
ST_STOP
@ ST_STOP
Definition: st.h:99
ST_CHECK
@ ST_CHECK
Definition: st.h:99
ST_DELETE
@ ST_DELETE
Definition: st.h:99
ST_REPLACE
@ ST_REPLACE
Definition: st.h:99
ST_CONTINUE
@ ST_CONTINUE
Definition: st.h:99
st_copy
#define st_copy
Definition: st.h:162
st_init_strcasetable_with_size
#define st_init_strcasetable_with_size
Definition: st.h:116
st_data_t
unsigned long st_data_t
Definition: st.h:22
st_init_numtable_with_size
#define st_init_numtable_with_size
Definition: st.h:108
st_foreach_check_callback_func
int st_foreach_check_callback_func(st_data_t, st_data_t, st_data_t, int)
Definition: st.h:138
st_locale_insensitive_strncasecmp
#define st_locale_insensitive_strncasecmp
Definition: st.h:170
st_hash_uint
#define st_hash_uint
Definition: st.h:180
st_get_key
#define st_get_key
Definition: st.h:130
st_numhash
#define st_numhash
Definition: st.h:166
st_foreach
#define st_foreach
Definition: st.h:142
st_insert2
#define st_insert2
Definition: st.h:126
st_numcmp
#define st_numcmp
Definition: st.h:164
st_hash
#define st_hash
Definition: st.h:176
st_cleanup_safe
#define st_cleanup_safe
Definition: st.h:158
st_init_strtable_with_size
#define st_init_strtable_with_size
Definition: st.h:112
st_locale_insensitive_strcasecmp
#define st_locale_insensitive_strcasecmp
Definition: st.h:168
st_hash_uint32
#define st_hash_uint32
Definition: st.h:178
st_values
#define st_values
Definition: st.h:150
st_lookup
#define st_lookup
Definition: st.h:128
st_init_table_with_size
#define st_init_table_with_size
Definition: st.h:104
st_values_check
#define st_values_check
Definition: st.h:152
st_clear
#define st_clear
Definition: st.h:160
st_keys
#define st_keys
Definition: st.h:146
ST_INDEX_BITS
#define ST_INDEX_BITS
Definition: st.h:66
st_add_direct
#define st_add_direct
Definition: st.h:154
st_delete
#define st_delete
Definition: st.h:118
st_memsize
#define st_memsize
Definition: st.h:174
SIZEOF_ST_INDEX_T
#define SIZEOF_ST_INDEX_T
Definition: st.h:59
st_hash_end
#define st_hash_end
Definition: st.h:182
st_foreach_check
#define st_foreach_check
Definition: st.h:144
st_shift
#define st_shift
Definition: st.h:122
st_update_callback_func
int st_update_callback_func(st_data_t *key, st_data_t *value, st_data_t arg, int existing)
Definition: st.h:131
st_insert
#define st_insert
Definition: st.h:124
st_index_t
st_data_t st_index_t
Definition: st.h:50
st_foreach_callback_func
int st_foreach_callback_func(st_data_t, st_data_t, st_data_t)
Definition: st.h:137
st_foreach_with_replace
#define st_foreach_with_replace
Definition: st.h:140
st_init_table
#define st_init_table
Definition: st.h:102
st_free_table
#define st_free_table
Definition: st.h:156
st_update
#define st_update
Definition: st.h:136
st_keys_check
#define st_keys_check
Definition: st.h:148
_
#define _(args)
Definition: stdarg.h:31
strlen
size_t strlen(const char *)
__sFILE
Definition: vsnprintf.c:169
bin
Definition: gzjoin.c:78
functor
Definition: hash.c:925
functor::func
st_foreach_callback_func * func
Definition: hash.c:926
functor::arg
st_data_t arg
Definition: hash.c:927
ind
Definition: gun.c:81
st_features
Definition: st.c:190
st_features::entry_power
unsigned char entry_power
Definition: st.c:192
st_features::size_ind
unsigned char size_ind
Definition: st.c:198
st_features::bin_power
unsigned char bin_power
Definition: st.c:196
st_features::bins_words
st_index_t bins_words
Definition: st.c:201
st_hash_type
Definition: st.h:61
st_table_entry
Definition: st.c:133
st_table_entry::hash
st_hash_t hash
Definition: st.c:134
st_table_entry::record
st_data_t record
Definition: st.c:136
st_table_entry::key
st_data_t key
Definition: st.c:135
st_table
Definition: st.h:79
st_table::bin_power
unsigned char bin_power
Definition: st.h:81
st_table::entries
st_table_entry * entries
Definition: st.h:94
st_table::bins
st_index_t * bins
Definition: st.h:88
st_table::entry_power
unsigned char entry_power
Definition: st.h:81
st_table::num_entries
st_index_t num_entries
Definition: st.h:86
st_table::size_ind
unsigned char size_ind
Definition: st.h:81
tab
Definition: enough.c:118
snprintf
#define snprintf
Definition: subst.h:14
t
#define t
Definition: symbol.c:253
VALUE
unsigned long VALUE
Definition: value.h:38
getenv
#define getenv(name)
Definition: win32.c:80