mdbm.h
Go to the documentation of this file.
1 /* Copyright 2013 Yahoo! Inc. */
2 /* See LICENSE in the root of the distribution for licensing details. */
3 
4 /* Modified by Yahoo! Inc.
5  *
6  * Modifications for:
7  * . Large object storage
8  * . Overflow pages
9  * . Windowed mode access
10  * . Hash-based intra-page scanning
11  * By Rick Reed
12  *
13  * Conversion from proprietary to PThreads based locking by Tim Crowder.
14  *
15  * Modifications for:
16  * . Extensive documentation
17  * . Many added tools
18  * . Extensive unit/functional tests
19  * . Many bug fixes (valgrind clean!)
20  * . Performance enhancements
21  * By
22  * Steve Carney
23  * Tim Crowder
24  * Max Kislik
25  * Mark Lakes
26  * Simon Baby
27  * Bhagyashri Mahule
28  * Lakshmanan Suryanarayanan
29  *
30  */
31 
32 /*
33  * Based on:
34  * mdbm - ndbm work-alike hashed database library based on sdbm which is
35  * based on Per-Aake Larson's Dynamic Hashing algorithms.
36  * BIT 18 (1978).
37  *
38  * sdbm Copyright (c) 1991 by Ozan S. Yigit (oz@nexus.yorku.ca)
39  *
40  * Modifications that:
41  * . Allow 64 bit sized databases,
42  * . used mapped files & allow multi reader/writer access,
43  * . move directory into file, and
44  * . use network byte order for db data structures.
45  * . support fixed size db (with shake function support)
46  * . selectable hash functions
47  * . changed page layout to support data alignment
48  * . support btree pre-split and tree merge/compress
49  * . added a mdbm checker (cf fsck)
50  * . add a statistic/profiler function (for tuning)
51  * . support mdbm_firstkey(), mdbm_nextkey() call.
52  * are:
53  * mdbm Copyright (c) 1995, 1996 by Larry McVoy, lm@sgi.com.
54  * mdbm Copyright (c) 1996 by John Schimmel, jes@sgi.com.
55  * mdbm Copyright (c) 1996 by Andrew Chang awc@sgi.com
56  *
57  * Modification that
58  * support NT/WIN98/WIN95 WIN32 environment
59  * support memory only (non-mmaped) database
60  * are
61  * mdbm Copyright (c) 1998 by Andrew Chang awc@bitmover.com
62  *
63  * Permission to use, copy, modify, and distribute this software and its
64  * documentation for any purpose and without fee is hereby granted,
65  * provided that the above copyright notice appear in all copies and that
66  * both that copyright notice and this permission notice appear in
67  * supporting documentation.
68  *
69  * This file is provided AS IS with no warranties of any kind. The author
70  * shall have no liability with respect to the infringement of copyrights,
71  * trade secrets or any patents by this file or any part thereof. In no
72  * event will the author be liable for any lost revenue or profits or
73  * other special, indirect and consequential damages.
74  */
75 #ifndef __MDBM_H_
76 #define __MDBM_H_
77 #ifdef __cplusplus
78 extern "C" {
79 #endif
80 
81 #define MDBM_API_VERSION 4
82 
83 #include <fcntl.h>
84 #include <inttypes.h>
85 #include <sys/types.h>
86 #include <time.h>
87 #include <stdio.h>
88 
89 
90 /*
91  * File format.
92  *
93  * The database is divided up into m_psize "pages". The pages do not have
94  * to be the same size as the system operating page size, but it is a good
95  * idea to make one a multiple of the other. The database won't create
96  * pages smaller than 128 bytes. Datums must fit on a single page so you
97  * probably don't want to change this. We default to 4K page size if no
98  * blocksize is specified at create time. The page size can't be bigger
99  * than 64K because we use 16 bit offsets in the pages.
100  *
101  * Page types. The first page is special. It contains the database header,
102  * as well as the first set of directory bits. The directory bits occupy
103  * 8 bytes.The point of putting the directory in
104  * the first page was to provide for a very small memory footprint for
105  * small databases; this allows this code to be used as a generic hashing
106  * package.
107  *
108  * After the database reaches a size where the directory will no longer fit
109  * in the 8 bytes of the first page, the data & directory are grown by
110  * doubling its previous size.
111  * The directory is copied to the end of data area. The default page
112  * get you to to a terabyte (2^40) database before copying
113  * becomes a problem.
114  *
115  * Page format:
116  * +--------+-----+----+----------+
117  * | key | data | key |
118  * +--------+----------+----------+
119  * | <---- - - - | data |
120  * +--------+-----+----+----------+
121  * | F R E E A R E A |
122  * +--------------+---------------+
123  * ino | n | keyoff | datoff | keyoff |
124  * +------------+--------+--------+
125  *
126  * We hide our own data in a page by making keyoff == 0. We do not allow
127  * 0 length keys as a result.
128  * Suppose we have a 1024 byte page, the header is 14 bytes
129  * (i.e. sizeof(MDBM_hdr)),
130  * the directory is 8 bytes. (i.e. total is 14 + 8 = 22 bytes )
131  *
132  * Page 0 looks like
133  * +--------+-----+----+----------+ first entry in page zero
134  * | 14 bytes header (never moves)| has a zero length key.
135  * | and 8 bytes for directory | data area has a mdbm header
136  * +--------+-----+----+----------+ and a directory bit array.
137  * | F R E E A R E A |
138  * +--------+-----+----+----------+
139  * ino | 2 | 0 | 22 | |
140  * +------------+--------+--------+
141  * ^^^
142  * this signifies a zero length key
143  */
144 
145 
146 /*
147  * If your enviromnet supports 64 bit int
148  * define the following
149  */
150 /* #define SUPPORT_64BIT */
151 
152 #if 1
153 #define mdbm_big_t int32_t
154 #define mdbm_ubig_t uint32_t
155 #else
156 #ifdef SUPPORT_64BIT
157 #define big int64_t
158 #define ubig uint64_t
159 #else
160 #define big int32_t
161 #define ubig uint32_t
162 #endif
163 #endif
164 
165 /*#define mdbm_ubig_t uint32_t */
166 /*#define mdbm_ubig_t unsigned int */
167 
168 typedef struct {
169  char *dptr; /**< Pointer to key or value data */
170  int dsize; /**< Number of bytes */
171 } datum;
172 
173 typedef struct {
174  datum key; /**< Key to access a database record */
175  datum val; /**< Value associated with the above key */
176 } kvpair;
177 
178 #define MDBM_KEYLEN_MAX (1<<15) /**< Maximum key size */
179 #define MDBM_VALLEN_MAX (1<<24) /**< Maximum key size */
180 
181 struct mdbm_iter {
182  mdbm_ubig_t m_pageno; /**< Last page fetched */
183  int m_next; /**< Index for getnext, or -1 */
184 };
185 
186 typedef struct mdbm_iter MDBM_ITER;
187 
188 /**
189  * Initializes an MDBM iterator.
190  * \param[in] i L-Value reference
191  * Usage: MDBM_ITER_INIT (MDBM_ITER *i);
192  */
193 #define MDBM_ITER_INIT(i) { (i)->m_pageno = 0; (i)->m_next = -1; }
194 /** MDBM iterator static initialization */
195 #define MDBM_ITER_INITIALIZER { 0, -1 }
196 
197 #define MDBM_LOC_NORMAL 0
198 #define MDBM_LOC_ARENA 1
199 
200 
201 typedef struct mdbm MDBM;
202 
203 /**
204  * \defgroup FileManagementGroup File Management Group
205  * The File Management Group contains the API the affects an MDBM at a file level.
206  * \{
207  */
208 
209 /*
210  * mdbm_open flags
211  */
212 
213 #define MDBM_O_RDONLY 0x00000000 /**< Read-only access */
214 #define MDBM_O_WRONLY 0x00000001 /**< Write-only access (deprecated in V3) */
215 #define MDBM_O_RDWR 0x00000002 /**< Read and write access */
216 #define MDBM_O_ACCMODE (MDBM_O_RDONLY | MDBM_O_WRONLY | MDBM_O_RDWR)
217 
218 #ifdef __linux__
219 #define MDBM_O_CREAT 0x00000040 /**< Create file if it does not exist */
220 #define MDBM_O_TRUNC 0x00000200 /**< Truncate file */
221 #define MDBM_O_FSYNC 0x00001000 /**< Sync file on close */
222 #define MDBM_O_ASYNC 0x00002000 /**< Perform asynchronous writes */
223 #else
224 #define MDBM_O_ASYNC 0x00000040 /**< Perform asynchronous writes */
225 #define MDBM_O_FSYNC 0x00000080 /**< Sync file on close */
226 #define MDBM_O_CREAT 0x00000200 /**< Create file if it does not exist */
227 #define MDBM_O_TRUNC 0x00000400 /**< Truncate file */
228 #endif
229 #define MDBM_O_DIRECT 0x00004000 /**< Perform direction I/O */
230 
231 #define MDBM_NO_DIRTY 0x00010000 /**< Do not not track clean/dirty status */
232 #define MDBM_SINGLE_ARCH 0x00080000 /**< User *promises* not to mix 32/64-bit access */
233 #define MDBM_OPEN_WINDOWED 0x00100000 /**< Use windowing to access db */
234 #define MDBM_PROTECT 0x00200000 /**< Protect database except when locked */
235 #define MDBM_DBSIZE_MB 0x00400000 /**< Dbsize is specific in MB */
236 #define MDBM_STAT_OPERATIONS 0x00800000 /**< collect stats for fetch, store, delete */
237 #define MDBM_LARGE_OBJECTS 0x01000000 /**< Support large objects - obsolete */
238 #define MDBM_PARTITIONED_LOCKS 0x02000000 /**< Partitioned locks */
239 #define MDBM_RW_LOCKS 0x08000000 /**< Read-write locks */
240 #define MDBM_ANY_LOCKS 0x00020000 /**< Open, even if existing locks don't match flags */
241 /* #define MDBM_CREATE_V2 0x10000000 / * create a V2 db */
242 #define MDBM_CREATE_V3 0x20000000 /**< Create a V3 db */
243 /* Flag is reserved 0x40000000 */
244 #define MDBM_OPEN_NOLOCK 0x80000000 /**< Don't lock during open */
245 
246 #define MDBM_DEMAND_PAGING 0x04000000 /**< (v2 only) */
247 #define MDBM_DBSIZE_MB_OLD 0x04000000 /**< (don't use -- conflicts with above) */
248 
249 
250 /* sanity checks */
251 
252 #if MDBM_O_RDONLY != O_RDONLY
253 #error must have MDBM_O_RDONLY == O_RDONLY
254 #endif
255 #if MDBM_O_WRONLY != O_WRONLY
256 #error must have MDBM_O_WRONLY == O_WRONLY
257 #endif
258 #if MDBM_O_RDWR != O_RDWR
259 #error must have MDBM_O_RDWR == O_RDWR
260 #endif
261 #if MDBM_O_ACCMODE != O_ACCMODE
262 #error must have MDBM_O_ACCMODE == O_ACCMODE
263 #endif
264 #if MDBM_O_CREAT != O_CREAT
265 #error must have MDBM_O_CREAT == O_CREAT
266 #endif
267 #if MDBM_O_TRUNC != O_TRUNC
268 #error must have MDBM_O_TRUNC == O_TRUNC
269 #endif
270 #if MDBM_O_ASYNC != O_ASYNC
271 #error must have MDBM_O_ASYNC == O_ASYNC
272 #endif
273 #if MDBM_O_FSYNC != O_FSYNC
274 #if __GNUC__ < 4
275 /* NOTE: on RHEL6 these are not equal, this is a known problem. */
276 #error must have MDBM_O_FSYNC == O_FSYNC
277 #endif
278 #endif
279 
280 /**
281  * Creates and/or opens a database
282  *
283  * \param[in] file Name of the backing file for the database.
284  * \param[in] flags Specifies the open-mode for the file, usually either
285  * (MDBM_O_RDWR|MDBM_O_CREAT) or (MDBM_O_RDONLY). Flag
286  * MDBM_LARGE_OBJECTS may be used to enable large object support.
287  * Large object support can only be enabled when the database is first
288  * created. Subsequent mdbm_open calls will ignore the flag.
289  * Flag MDBM_PARTITIONED_LOCKS may be used to enable
290  * partition locking a per mdbm_open basis.
291  * \param[in] mode Used to set the file permissions if the file needs to
292  * be created.
293  * \param[in] psize Specifies the page size for the database and is set
294  * when the database is created.
295  * The minimum page size is 128.
296  * In v2, the maximum is 64K.
297  * In v3, the maximum is 16M - 64.
298  * The default, if 0 is specified, is 4096.
299  * \param[in] presize Specifies the initial size for the database. The
300  * database will dynamically grow as records are added, but specifying
301  * an initial size may improve efficiency. If this is not a multiple
302  * of \a psize, it will be increased to the next \a psize multiple.
303  * \return MDBM handle or failure indicator
304  * \retval Pointer to an MDBM handle, on success
305  * \retval NULL on error, and errno is set
306  *
307  * Values for \a flags mask:
308  * - MDBM_O_RDONLY - open for reading only
309  * - MDBM_O_RDWR - open for reading and writing
310  * - MDBM_O_WRONLY - same as RDWR (deprecated)
311  * - MDBM_O_CREAT - create file if it does not exist (requires flag MDBM_O_RDWR)
312  * - MDBM_O_TRUNC - truncate size to 0
313  * - MDBM_O_ASYNC - enable asynchronous sync'ing by the kernel syncing process.
314  * - MDBM_O_FSYNC - sync MDBM upon \ref mdbm_close
315  * - MDBM_O_DIRECT - use O_DIRECT when accessing backing-store files
316  * - MDBM_NO_DIRTY - do not not track clean/dirty status
317  * - MDBM_OPEN_WINDOWED - use windowing to access db, only available with MDBM_O_RDWR
318  * - MDBM_PROTECT - protect database except when locked (MDBM V3 only)
319  * - MDBM_DBSIZE_MB - dbsize is specific in MB (MDBM V3 only)
320  * - MDBM_LARGE_OBJECTS - support large objects
321  * - MDBM_PARTITIONED_LOCKS - partitioned locks
322  * - MDBM_RW_LOCKS - read-write locks
323  * - MDBM_CREATE_V2 - create a V2 db
324  * - MDBM_CREATE_V3 - create a V3 db
325  * - MDBM_HEADER_ONLY - map header only (internal use)
326  * - MDBM_OPEN_NOLOCK - do not lock during open
327  * - MDBM_ANY_LOCKS - (V4 only) treat the locking flags as only a suggestion
328  * - MDBM_SINGLE_ARCH - (V4 only) user guarantees no mixed (32/64-bit) access
329  * in exchange for faster (pthreads) locking
330  *
331  * More information about the differences between V2 and V3 mdbm_open behavior:
332  * - In V2, when the \a psize (page size) parameter is invalid, the page size
333  * is silently converted to a valid value: if non-zero but less than 128, it
334  * becomes 128. Page size is always rounded to the next power of 2. If
335  * greater than 64K, it becomes 64K.
336  * - In V3, when the page size is invalid(less than 128, or greater than 16MB)
337  * mdbm_open detects an error and returns NULL. Page size is always a
338  * multiple of 64 bytes.
339  *
340  */
341 extern MDBM* mdbm_open(const char *file, int flags, int mode, int psize, int presize);
342 
343 /**
344  * Closes the database.
345  *
346  * \param[in,out] db Database handle
347  *
348  * mdbm_close closes the database specified by the \a db argument. The
349  * in-memory pages are not flushed to disk by close. They will be written to
350  * disk over time as they are paged out, but an explicit \ref mdbm_sync call is
351  * necessary before closing if on-disk consistency is required.
352  */
353 extern void mdbm_close(MDBM *db);
354 
355 /**
356  * Closes an MDBM's underlying file descriptor.
357  *
358  * \param[in,out] db Database handle
359  */
360 extern void mdbm_close_fd(MDBM *db);
361 
362 /**
363  * Duplicate an existing database handle. The advantage of dup'ing a handle
364  * over doing a separate \ref mdbm_open is that dup's handle share the same virtual
365  * page mapping within the process space (saving memory).
366  * Threaded applications should use pthread_mutex_lock and unlock around calls to mdbm_dup_handle.
367  *
368  * \param[in,out] db Database handle
369  * \param[in] flags Reserved for future use
370  * \return Dup'd handle
371  * \retval Pointer to an MDBM handle, on success
372  * \retval NULL on error, and errno is set
373  */
374 extern MDBM* mdbm_dup_handle(MDBM* db, int flags);
375 
376 /**
377  * msync's all pages to disk asynchronously. The mdbm_sync call will return, and
378  * mapped pages are scheduled to be flushed to disk.
379  *
380  * \param[in,out] db Database handle
381  * \return Sync status
382  * \retval -1 Error, and errno is set
383  * \retval 0 Success
384  */
385 extern int mdbm_sync(MDBM *db);
386 
387 /**
388  * fsync's an MDBM. Syncs all pages to disk synchronously. The mdbm_fsync call
389  * will return after all pages have been flushed to disk. The database is locked
390  * while pages are flushed.
391  *
392  * \param[in,out] db Database handle
393  * \return fsync status
394  * \retval -1 Error, and errno is set
395  * \retval 0 Success
396  */
397 extern int mdbm_fsync(MDBM *db);
398 
399 /**
400  * Atomically replaces the database currently in oldfile \a db with the new
401  * database in \a newfile. The old database is locked while the new database
402  * is renamed from newfile to oldfile and the old database is marked as having
403  * been replaced. This causes all processes that have the old database open
404  * to reopen the new database on their next access. Only database files of
405  * the same version may be specified for oldfile and new file. For example,
406  * mix and matching of v2 and v3 with oldfile and newfile is not allowed.
407  *
408  * This function will delete the old file; and rename the new file.
409  *
410  * \param[in,out] db Database handle
411  * \param[in] newfile Path of new file
412  * \return Replace status
413  * \retval -1 Error and errno is set. The MDBM was not replaced.
414  * \retval 0 Success and the MDBM was replaced
415  */
416 extern int mdbm_replace_db(MDBM* db, const char* newfile);
417 
418 /**
419  * Atomically replaces an old database in \a oldfile with a new database in
420  * \a newfile. \a oldfile is deleted, and \a newfile is renamed to \a oldfile.
421  *
422  * The old database is locked (if the MDBM were opened with locking) while the
423  * new database is renamed from \a newfile to \a oldfile, and the old database
424  * is marked as having been replaced. The marked old database causes all
425  * processes that have the old database open to reopen using the new database on
426  * their next access.
427  *
428  * Only database files of the same version may be specified for \a oldfile and
429  * \a newfile. For example, mixing and matching of v2 and v3 for \a oldfile and
430  * \a newfile is not allowed.
431  *
432  * mdbm_replace_file may be used if the MDBM is opened with locking or without
433  * locking (using mdbm_open flag MDBM_OPEN_NOLOCK), and without per-access
434  * locking, if all accesses are read (fetches) accesses across all programs that
435  * open that MDBM. If there are any write (store/delete) accesses, you must
436  * open the MDBM with locking, and you must lock around all operations (fetch,
437  * store, delete, iterate).
438  *
439  * \param[in] oldfile File to be replaced
440  * \param[in] newfile Path of new file
441  * \return Replace status
442  * \retval -1 Error, and errno is set. The MDBM was not replaced
443  * \retval 0 Success, and the MDBM was replaced
444  */
445 extern int mdbm_replace_file(const char* oldfile, const char* newfile);
446 
447 /**
448  * Forces a db to split, creating N pages. Must be called before any data is
449  * inserted. If N is not a multiple of 2, it will be rounded up.
450  *
451  * \param[in,out] db Database handle
452  * \param[in] N Target number of pages post split.
453  * If \a N is not larger than the initial size (ex., 0),
454  * a split will not be done and a success status is returned.
455  * \return Split status
456  * \retval -1 Error, and errno is set. errno=EFBIG if MDBM not empty.
457  * \retval 0 Success
458  */
459 extern int mdbm_pre_split(MDBM *db, mdbm_ubig_t N);
460 
461 #define MDBM_COPY_LOCK_ALL 0x01
462 
463 /**
464  * Copies the contents of a database to an open file handle.
465  *
466  * \param[in,out] db Database handle
467  * \param[in] fd Open file descriptor
468  * \param[in] flags
469  * \return fcopy status
470  * \retval -1 Error, and errno is set
471  * \retval 0 Success
472  *
473  * Values for \a flags mask:
474  * - MDBM_COPY_LOCK_ALL - Whether lock for the duration of the copy.
475  * For a consistent snapshot of the entire database, this flag must be
476  * used. Otherwise, consistency will only be on a per-page level.
477  */
478 extern int mdbm_fcopy(MDBM* db, int fd, int flags);
479 
480 
481 #define MDBM_SAVE_COMPRESS_TREE 0x01000000 /* compress mdbm before saving */
482 
483 /**
484  * DEPRECATED!! mdbm_save is only supported for V2 MDBMs.
485  * Saves the current contents of the database in a transportable format (i.e.,
486  * without all of the holes normally found in an MDBM database). Include
487  * MDBM_O_CREAT in flags to create a new save file, MDBM_O_TRUNC to truncate
488  * an existing file. Include both to either create a new file or truncate an
489  * existing one, as required.
490  *
491  * \param[in,out] db Database handle
492  * \param[in] file File to save to
493  * \param[in] flags Mask indicating how file should be created Supported flags
494  * are: MDBM_O_CREAT, MDBM_O_TRUNC, MDBM_SAVE_COMPRESS_TREE(used to compress
495  * a database internally before saving to file)
496  * \param[in] mode Set as file mode permissions (ex: 0644); If 0 then ignored
497  * \param[in] compressionLevel If <= 0, then file is saved without compressing.
498  * If > zero, zlib compression at the specified level is applied
499  * \return Save status
500  * \retval 0 Success
501  * \retval -1 Error, and errno is set
502  */
503 extern int mdbm_save(MDBM *db, const char *file, int flags, int mode, int compressionLevel);
504 
505 /**
506  * DEPRECATED!! mdbm_restore is only supported for V2 MDBMs.
507  * Restores a database from the specified file (which was created using the
508  * \ref mdbm_save function). Any existing contents in the database are truncated
509  * before restoring.
510  *
511  * \param[in,out] db Database handle
512  * \param[in] file File to restore from
513  * \return Restore status
514  * \retval 0 Success
515  * \retval -1 Error, and errno is set
516  */
517 extern int mdbm_restore(MDBM *db, const char *file);
518 
519 /** \} FileManagementGroup */
520 
521 
522 
523 /**
524  * \defgroup ConfigurationGroup Configuration Group
525  * The Configuration Group contains the API the gets/sets the underlying
526  * configuration of an MDBM. Most of the commands that set the configuration
527  * must be done at MDBM-creation time.
528  * \{
529  */
530 
531 /**
532  * Prints to stdout various pieces of information, specifically: page size,
533  * page count, hash function, running with or without locking
534  *
535  * NOTE: There is only a V2 implementation. V3 not currently supported.
536  *
537  * \param[in,out] db Database handle
538  */
539 extern void mdbm_stat_header(MDBM *db);
540 
541 /**
542  * Gets the MDBM's hash function identifier.
543  *
544  * \param[in,out] db Database handle
545  * \return Hash function identifier
546  * \retval MDBM_HASH_CRC32 - Table based 32bit CRC
547  * \retval MDBM_HASH_EJB - From hsearch
548  * \retval MDBM_HASH_PHONG - Congruential hash
549  * \retval MDBM_HASH_OZ - From sdbm
550  * \retval MDBM_HASH_TOREK - From BerkeleyDB
551  * \retval MDBM_HASH_FNV - Fowler/Vo/Noll hash (DEFAULT)
552  * \retval MDBM_HASH_STL - STL string hash
553  * \retval MDBM_HASH_MD5 - MD5
554  * \retval MDBM_HASH_SHA_1 - SHA_1
555  * \retval MDBM_HASH_JENKINS - Jenkins string
556  * \retval MDBM_HASH_HSIEH - Hsieh SuperFast
557  */
558 extern int mdbm_get_hash(MDBM *db);
559 
560 /**
561  * Sets the hashing function for a given MDBM. The hash function must be set
562  * before storing anything to the db (this is not enforced, but entries stored
563  * before the hash change will become inaccessible if the hash function is
564  * changed).
565  *
566  * NOTE: setting the hash must be be done at creation time, or when there is
567  * no data in an MDBM. Changing the hash function when there is existing data
568  * will result in not being able access that data in the future.
569  *
570  * \param[in,out] db Database handle
571  * \param[in] hashid Numeric identifier for new hash function.
572  * \return Set hash status
573  * \retval 0 Error, invalid \a hashid
574  * \retval 1 Success
575  *
576  * Available Hash IDs are:
577  * - MDBM_HASH_CRC32 - Table based 32bit CRC
578  * - MDBM_HASH_EJB - From hsearch
579  * - MDBM_HASH_PHONG - Congruential hash
580  * - MDBM_HASH_OZ - From sdbm
581  * - MDBM_HASH_TOREK - From BerkeleyDB
582  * - MDBM_HASH_FNV - Fowler/Vo/Noll hash (DEFAULT)
583  * - MDBM_HASH_STL - STL string hash
584  * - MDBM_HASH_MD5 - MD5
585  * - MDBM_HASH_SHA_1 - SHA_1
586  * - MDBM_HASH_JENKINS - Jenkins string
587  * - MDBM_HASH_HSIEH - Hsieh SuperFast
588  */
589 extern int mdbm_sethash(MDBM *db, int hashid);
590 
591 
592 /**
593  * Sets the size of item data value which will be put on the large-object heap
594  * rather than inline. The spill size can be changed at any point after the
595  * db has been created. However, it's a recommended practice to set the spill
596  * size at creation time.
597  *
598  * NOTE: The database has to be opened with the MDBM_LARGE_OBJECTS flag for
599  * spillsize to take effect.
600  *
601  * \param[in,out] db Database handle
602  * \param[in] size New large-object threshold size
603  * \return Set spill size status
604  * \retval -1 Error, and errno is set
605  * \retval 0 Success
606  */
607 extern int mdbm_setspillsize(MDBM* db, int size);
608 
609 
610 /**
611  * Sets the size of item data value which will be put on the large-object heap
612  * rather than inline. The spill size can be changed at any point after the
613  * db has been created. However, it's a recommended practice to set the spill
614  * size at creation time.
615  *
616  * NOTE: The database has to be opened with the MDBM_LARGE_OBJECTS flag for
617  * spillsize to take effect.
618  *
619  * \param[in,out] db Database handle
620  * \param[in] size New large-object threshold size
621  * \return Set spill size status
622  * \retval -1 Error, and errno is set
623  * \retval 0 Success
624  */
625 extern int mdbm_setspillsize(MDBM* db, int size);
626 
627 
628 /*
629  * mdbm_set_alignment flags
630  */
631 #define MDBM_ALIGN_8_BITS 0x0 /**< 1-Byte data alignment */
632 #define MDBM_ALIGN_16_BITS 0x1 /**< 2-Byte data alignment */
633 #define MDBM_ALIGN_32_BITS 0x3 /**< 4-Byte data alignment */
634 #define MDBM_ALIGN_64_BITS 0x7 /**< 8-Byte data alignment */
635 
636 /**
637  * Gets the MDBM's record byte-alignment.
638  *
639  * \param[in,out] db Database handle
640  * \return Alignment mask.
641  * \retval 0 - 8-bit alignment
642  * \retval 1 - 16-bit alignment
643  * \retval 3 - 32-bit alignment
644  * \retval 7 - 64-bit alignment
645  */
646 extern int mdbm_get_alignment(MDBM *db);
647 
648 /**
649  * Sets a database's byte-size alignment for keys and values within a page.
650  * This feature is useful for hardware/memory architectures that incur a
651  * performance penalty for unaligned accesses. Later (2006+) i386 and x86
652  * architectures do not need special byte alignment, and should use the
653  * default of 8-bit alignment.
654  *
655  * NOTE: setting the byte alignment must be be done at MDBM-creation time, or
656  * when there is no data in an MDBM. Changing the byte alignment when there
657  * is existing data will result in in undefined behavior and probably a crash.
658  *
659  * \param[in,out] db Database handle
660  * \param[in] align Alignment mask
661  * \return Alignment status
662  * \retval -1 Error, and errno is set
663  * \retval 0 Success
664  */
665 extern int mdbm_set_alignment(MDBM *db, int align);
666 
667 /**
668  * Gets the MDBM's size limit. Returns the limit set for the size of the db
669  * using the \ref mdbm_limit_size_v3 routine.
670  *
671  * \param[in,out] db Database handle
672  * \return database size limit
673  * \retval 0 No limit is set
674  * \retval Total number of bytes for maximum database size, including header and directory
675  */
676 extern uint64_t mdbm_get_limit_size(MDBM *db);
677 
678 /*
679  * Shake (page overflow) support
680  */
681 
682 /* NOT USED IN V4 */
683 /* struct mdbm_shake_data { */
684 /* uint32_t page_num; */ /* index number of overflowing page */
685 /* const char* page_begin; */ /* beginning address of page */
686 /* const char* page_end; */ /* one byte past last byte of page */
687 /* uint16_t page_free_space; */ /* current free space on page */
688 /* uint16_t space_needed; */ /* space needed for current insert */
689 /* uint16_t page_num_items; */ /* number of items on page */
690 /* uint16_t reserved; */
691 /* kvpair* page_items; */ /* key/val pairs for all items on page */
692 /* uint16_t* page_item_sizes; */ /* total in-page size for each item on page */
693 /* void* user_data; */ /* user-data pointer passed to mdbm_limit_size_new */
694 /* }; */
695 /*typedef int (*mdbm_shake_func)(MDBM *, const datum*, const datum*, struct mdbm_shake_data *);
696  extern int mdbm_limit_size(MDBM *, mdbm_ubig_t,
697  int (*func)(struct mdbm *, datum, datum, void *));
698  extern int mdbm_limit_size_new(MDBM *, mdbm_ubig_t, mdbm_shake_func shake, void *);
699 */
700 
702  uint32_t page_num; /**< Index number of overflowing page */
703  const char* page_begin; /**< Beginning address of page */
704  const char* page_end; /**< One byte past last byte of page */
705  uint32_t page_free_space; /**< Current free space on page */
706  uint32_t space_needed; /**< Space needed for current insert */
707  uint32_t page_num_items; /**< Number of items on page */
708  uint32_t reserved; /**< Reserved */
709  kvpair* page_items; /**< Key-Value pairs for all items on page */
710  uint32_t* page_item_sizes; /**< Total in-page size for each item on page */
711  void* user_data; /**< User-data pointer passed to \ref mdbm_limit_size_v3 */
712 };
713 
714 typedef int (*mdbm_shake_func_v3)(MDBM *, const datum*, const datum*,
715  struct mdbm_shake_data_v3 *);
716 /**
717  * Limits the size of a V3 database to a maximum number of pages. This
718  * function causes the MDBM to shake whenever a full page would cause the db
719  * to grow beyond the specified max number of pages. The limit size for an
720  * MDBM may be increased over time, but is may never be decreased.
721  *
722  * \param[in,out] db Database handle
723  * \param[in] max_page Maximum number of data pages
724  * \param[in] shake Shake function.
725  * \param[in] user Pointer to user data that will be passed to the \a shake function
726  * \return Limit size status
727  * \retval -1 Error, and errno is set
728  * \retval 0 Success
729  */
730 extern int mdbm_limit_size_v3(MDBM *db, mdbm_ubig_t max_page, mdbm_shake_func_v3 shake,void *user);
731 
732 /**
733  * Limits the internal page directory size to a number of \a pages. The number
734  * of pages is rounded up to a power of 2.
735  *
736  * \param[in,out] db Database handle
737  * \param[in] pages Number of data pages
738  * \return Limit dir size status
739  * \retval -1 Error, and errno is set
740  * \retval 0 Success
741  */
742 extern int mdbm_limit_dir_size(MDBM *db, int pages);
743 
744 /**
745  * Gets the on-disk format version number of an MDBM.
746  *
747  * \param[in,out] db Database handle
748  * \return On-disk file format version number
749  * \retval 2 - MDBM V2
750  * \retval 3 - MDBM V3
751  */
752 extern uint32_t mdbm_get_version (MDBM* db);
753 
754 /**
755  * Gets the current MDBM's size.
756  *
757  * \param[in,out] db Database handle
758  * \return Size of database in bytes.
759  */
760 extern uint64_t mdbm_get_size(MDBM *db);
761 
762 /**
763  * Get the MDBM's page size.
764  *
765  * \param[in,out] db Database handle
766  * \return Number of bytes in a database page
767  */
768 extern int mdbm_get_page_size(MDBM *db);
769 
770 /*
771  * Magic number which is also a version number. If the database format
772  * changes then we need another magic number.
773  *
774  * FIX: The magic number should be invariant across MDBM format versions
775  * because the number in /usr/share/file/magic should identify a general class
776  * of files. The version number should immediately follow the magic number.
777  * The magic number handling will have to change to reflect this fix.
778  * --carney
779  */
780 #define _MDBM_MAGIC 0x01023962 /**< V2 file identifier */
781 #define _MDBM_MAGIC_NEW 0x01023963 /**< V2 file identifier with large objects */
782 #define _MDBM_MAGIC_NEW2 0x01023964 /**< V3 file identifier */
783 
784 #define MDBM_MAGIC _MDBM_MAGIC_NEW2
785 
786 /**
787  * Gets the magic number from an MDBM.
788  *
789  * \param[in,out] db Database handle
790  * \param[out] magic MDBM magic number (internal version identifier)
791  * \return Magic number status
792  * \retval -3 Cannot read all of the magic number
793  * \retval -2 File is truncated (empty)
794  * \retval -1 Cannot read file
795  * \retval 0 Magic number returned in \a magic
796  */
797 extern int mdbm_get_magic_number(MDBM *db, uint32_t *magic);
798 
799 
800 /**
801  * Sets the window size for an MDBM. Windowing is typically used for a very
802  * large data store where only part of it will be mapped to memory. In
803  * windowing mode, pages are accessed through a "window" to the database.
804  *
805  * In order to use windowing mode, the page size must be a multiple of the
806  * system page size (4K on most systems). The system page size can be
807  * retrieved by calling getpagesize or sysconf(_SC_PAGESIZE).
808  *
809  * NOTE: It is best if the minimum window size chosen is:
810  * (4*max-large-object-size + n*8*pagesize)
811  *
812  * NOTE: API exists in MDBM V3 only, and using mdbm_set_window_size requires
813  * opening MDBM with MDBM_O_RDWR.
814  *
815  * \param[in,out] db Database handle for windowed access, typically a backing store database
816  * \param[in] wsize Window size which must be at least 4X the page size
817  * used for the database. 0 means to remove windowing
818  * \return Set window status
819  * \retval -1 Error, and errno is set
820  * \retval 0 Success
821  */
822 extern int mdbm_set_window_size(MDBM* db, size_t wsize);
823 
824 /** \} ConfigurationGroup */
825 
826 
827 /**
828  * \defgroup RecordAccessGroup Record Access Group
829  * The Record Access Group contains the API for fetching, storing, and
830  * deleting records.
831  * \{
832  */
833 
834 /**
835  * Fetches the record specified by the \a key argument and returns a \a datum
836  * that \b points to the value. If such a record exists in the database, the
837  * size and location (pointer) are stored in the returned \a datum. If no
838  * matching record exists, a null datum (dsize==0, dptr==NULL) is returned.
839  *
840  * The contents returned in \a datum has a <strong>pointer to a value</strong>
841  * in mapped memory. If there is any system-wide process that could modify your
842  * MDBM, this value must be accessed in a locked context.
843  *
844  * For example, the following is correct code to copy a lookup value for subsequent return:
845  * \code
846  * char* getFoo(int maxLength)
847  * {
848  * datum key = { "foo", 3 };
849  * datum value;
850  * char *buffer = (char*)malloc(maxLength+1); // Skip return code check.
851  * mdbm_lock(db); // Skip return code check.
852  * value = mdbm_fetch(db, &key);
853  * if (value.dptr != NULL) {
854  * strncpy(buffer, value.dptr, maxLength); // GOOD: copy-out done in locked context.
855  * } else {
856  * buffer[0] = '\0';
857  * };
858  * mdbm_unlock(db); // Skip return code check.
859  * return buffer;
860  * }
861  * \endcode
862  *
863  * \param[in,out] db Database handle
864  * \param[in] key Lookup key
865  * \return datum is returned with found, or not found, information
866  * \retval datum found, with size and pointer set, on success
867  * \retval datum not found, with dsize==0, and dptr==NULL,
868  * and errno==ENOENT. errno may have other values for failure (ex., EINVAL).
869  */
870 extern datum mdbm_fetch(MDBM *db, datum key);
871 
872 /**
873  * Fetches the record specified by the \a key argument. If such a record
874  * exists in the database, the size and location (pointer) are stored in the datum
875  * pointed to by \a val. If no matching record exists, a null datum (dsize==0,
876  * dptr==NULL) is returned.
877  *
878  * The contents returned in \a val is a <strong>pointer to a value</strong>
879  * in mapped memory. If there is any system-wide process that could modify your
880  * MDBM, this value must be accessed in a locked context.
881  *
882  * For example, the following is correct code to copy a lookup value for subsequent return:
883  * \code
884  * char* getFoo(int maxLength)
885  * {
886  * int rc;
887  * datum key = { "foo", 3 };
888  * datum value;
889  * char *buffer = (char*)malloc(maxLength+1); // Skip return code check.
890  * mdbm_lock(db); // Skip return code check.
891  * rc = mdbm_fetch_r(db, &key, &value, NULL);
892  * if (!rc) {
893  * strncpy(buffer, value.dptr, maxLength); // GOOD: copy-out done in locked context.
894  * } else {
895  * buffer[0] = '\0';
896  * };
897  * mdbm_unlock(db); // Skip return code check.
898  * return buffer;
899  * }
900  * \endcode
901  *
902  *
903  * A record can be updated in-place by fetching the value datum, casting the
904  * dptr as appropriate, and updating those contents. However, the update must
905  * not change the size of the record.
906  *
907  * \param[in,out] db Database handle
908  * \param[in] key Pointer to \ref datum used as the index key
909  * \param[out] val Pointer to \ref datum returned
910  * \param[in,out] iter MDBM iterator
911  * \return Fetch status
912  * \retval -1 Error, and errno is set
913  * \retval 0 Success
914  */
915 extern int mdbm_fetch_r(MDBM *db, datum *key, datum *val, MDBM_ITER *iter);
916 
917 /**
918  * Fetches and copies the record specified by the \a key argument. If such a
919  * record exists in the database, the size and location are stored in the
920  * datum pointed to by val. The same record is also copied into the \a buf
921  * argument. If no matching record exists, a null datum (dsize==0,
922  * dptr==NULL) is returned.
923  *
924  * Note that a record can be updated in-place by fetching the value datum,
925  * casting the dptr as appropriate, and updating the record. However, the
926  * update must not change the size of the record.
927  *
928  * buf.dptr must point to memory that has been previously malloc'd on the
929  * heap. buff.dptr will be realloc'd if is too small.
930  *
931  * \param[in,out] db Database handle
932  * \param[in] key Lookup key
933  * \param[out] val Lookup value (pointer)
934  * \param[in,out] buf Copy-out of lookup reference returned in \a val
935  * \param[in] flags Reserved for future use
936  * \return Fetch status
937  * \retval -1 Error, and errno is set
938  * \retval 0 Success
939  */
940 extern int mdbm_fetch_buf(MDBM *db, datum *key, datum *val, datum *buf, int flags);
941 
942 /**
943  * Fetches the next value for a key inserted via \ref mdbm_store_r with the
944  * MDBM_INSERT_DUP flag set. The order of values returned by iterating via
945  * this function is not guaranteed to be the same order as the values were
946  * inserted.
947  *
948  * As with any db iteration, record insertion and deletion during iteration
949  * may cause the iteration to skip and/or repeat records.
950  *
951  * Calling this function with an iterator initialized via \ref MDBM_ITER_INIT
952  * will cause this function to return the first value for the given key.
953  *
954  * \param[in,out] db Database handle
955  * \param[in] key Pointer to \ref datum used as the index key
956  * \param[out] val Pointer to \ref datum returned
957  * \param[in,out] iter MDBM iterator
958  * \return Fetch duplicate status
959  * \retval -1 Error, and errno is set
960  * \retval 0 Success
961  */
962 extern int mdbm_fetch_dup_r(MDBM *db, datum *key, datum *val, MDBM_ITER *iter);
963 
964 /**
965  * Fetches a string.
966  *
967  * \param[in,out] db Database handle
968  * \param[in] key Stored key
969  * \return Fetched string reference, or NULL
970  * \retval Pointer to string, on success
971  * \retval NULL if key does not exist
972  */
973 extern char* mdbm_fetch_str(MDBM *db, const char *key);
974 
975  /*
976  * MDBM fetch interface
977  */
978 #define MDBM_FETCH_FLAG_DIRTY 0x01 /**< Cache entry is dirty */
979 
981  uint32_t flags; /**< Entry flags */
982  uint32_t cache_num_accesses; /**< Number of accesses to cache entry */
983  uint32_t cache_access_time; /**< Last access time (LRU/LFU only) */
984 };
985 
986 /**
987  * Fetches and copies the record specified by the \a key argument. If such a
988  * record exists in the database, the size and location are stored in the
989  * datum pointed to by val. The same record is also copied into the \a buf
990  * argument. If no matching record exists, a null datum (dsize==0,
991  * dptr==NULL) is returned.
992  *
993  * \ref mdbm_fetch_info is only supported by MDBM version 3 or higher.
994  *
995  * Note that a record can be updated in-place by fetching the value datum,
996  * casting the dptr as appropriate, and updating the record. However, the
997  * update must not change the size of the record.
998  *
999  * Additional information is passed back in the \a info argument: entry flags,
1000  * the number of accesses to cache entry, and the last access time (LRU/LFU
1001  * only).
1002  *
1003  * buf.dptr cannot point to statically allocated memory. It must point to
1004  * memory that has been previously malloc'd on the heap.
1005  *
1006  * \param[in,out] db Database handle
1007  * \param[in] key Pointer to \ref datum used as the index key
1008  * \param[out] val Pointer to \ref datum returned
1009  * \param[out] buf Copy found \ref datum into this parameter
1010  * \param[out] info Pointer to additional information (struct mdbm_fetch_info)
1011  * \param[in,out] iter MDBM iterator
1012  * \return Fetch info status
1013  * \retval -1 Error, and errno is set
1014  * \retval 0 Success
1015  */
1016 extern int mdbm_fetch_info(MDBM *db, datum *key, datum *val, datum *buf,
1017  struct mdbm_fetch_info *info, MDBM_ITER *iter);
1018 
1019 /**
1020  * Deletes a specific record.
1021  *
1022  * \param[in,out] db Database handle
1023  * \param[in] key Key to be removed
1024  * \return Delete status
1025  * \retval -1 Error, and errno is set
1026  * \retval 0 Success
1027  */
1028 extern int mdbm_delete(MDBM *db, datum key);
1029 
1030 /**
1031  * Deletes the record currently addressed by the \a iter argument. After
1032  * deletion, the key and/or value returned by the iterating function is no
1033  * longer valid. Calling \ref mdbm_next_r on the iterator will return the
1034  * key/value for the entry following the entry that was deleted.
1035  *
1036  * \param[in,out] db Database handle
1037  * \param[in,out] iter MDBM iterator pointing to item to be deleted
1038  * \return Delete status
1039  * \retval -1 Error, and errno is set
1040  * \retval 0 Success
1041  */
1042 extern int mdbm_delete_r(MDBM *db, MDBM_ITER* iter);
1043 
1044 /**
1045  * Deletes a string from the MDBM.
1046  *
1047  * \param[in,out] db Database handle
1048  * \param[in] key Stored key
1049  * \return Delete string status
1050  * \retval -1 Error, and errno is set
1051  * \retval 0 Success
1052  */
1053 extern int mdbm_delete_str(MDBM *db, const char *key);
1054 
1055 
1056 /*
1057  * mdbm_store flags
1058  */
1059 #define MDBM_INSERT 0 /**< Insert if key does not exist; fail if exists */
1060 #define MDBM_REPLACE 1 /**< Update if key exists; insert if does not exist */
1061 #define MDBM_INSERT_DUP 2 /**< Insert new record (creates duplicate if key exists) */
1062 #define MDBM_MODIFY 3 /**< Update if key exists; fail if does not exist */
1063 #define MDBM_STORE_MASK 0x3 /**< Mask for all store options */
1064 
1065 /** Extract the store mode from an options mask */
1066 #define MDBM_STORE_MODE(f) ((f) & MDBM_STORE_MASK)
1067 
1068 #define MDBM_RESERVE 0x100 /**< Reserve space; Value not copied */
1069 #define MDBM_CLEAN 0x200 /**< Mark entry as clean */
1070 #define MDBM_CACHE_ONLY 0x400 /**< Do not operate on the backing store; use cache only */
1071 
1072 #define MDBM_CACHE_REPLACE 0 /**< Update cache if key exists; insert if does not exist */
1073 #define MDBM_CACHE_MODIFY 0x1000 /**< Update cache if key exists; do not insert if does not */
1074 #define MDBM_CACHE_UPDATE_MODE(f) ((f) & MDBM_CACHE_MODIFY)
1075 
1076 #define MDBM_STORE_SUCCESS 0 /**< Returned if store succeeds */
1077 #define MDBM_STORE_ENTRY_EXISTS 1 /**< Returned if MDBM_INSERT used and key exists */
1078 
1079 /**
1080  * This is a wrapper around \ref mdbm_store_r, with a static iterator.
1081  *
1082  * \param[in,out] db Database handle
1083  * \param[in] key Stored key
1084  * \param[in] val \a Key's value
1085  * \param[in] flags Store flags
1086  * \return Store status
1087  * \retval -1 Error, and errno is set
1088  * \retval 0 Success
1089  * \retval 1 Flag MDBM_INSERT was specified, and the key already exists
1090  *
1091  * Values for \a flags mask:
1092  * - MDBM_INSERT - Operation will fail if a record with the same key
1093  * already exists in the database.
1094  * - MDBM_REPLACE - A record with the same key will be replaced.
1095  * If the key does not exist, the record will be created.
1096  * - MDBM_INSERT_DUP - allows multiple records with the same key to be inserted.
1097  * Fetching a record with the key will return only one of the duplicate
1098  * records, and which record is returned is not defined.
1099  * - MDBM_MODIFY - Store only if matching entry already exists.
1100  * - MDBM_RESERVE - Reserve space; value not copied (\ref mdbm_store_r only)
1101  * - MDBM_CACHE_ONLY (MDBM V3) - Perform store only in the Cache, not in Backing Store.
1102  * - MDBM_CACHE_MODIFY (MDBM V3) - Update Cache only if key exists; update the Backing Store
1103  *
1104  * Insertion with flag \em MDBM_MODIFY set will fail if the key does not
1105  * already exist.
1106  *
1107  * Insertion with flag \em MDBM_RESERVE set will allocate space within the
1108  * MDBM for the object; but will not copy it. The caller is responsible for
1109  * copying the value into the indicated position.
1110  *
1111  * Replacement with flag \em MDBM_CACHE_MODIFY for a cache with a backing
1112  * store, has the effect of writing through the cache to the backing store
1113  * without updating the cache unless that cache entry already exists.
1114  *
1115  * Quick option summary:
1116  * - MDBM_INSERT - Add if missing, fail if present
1117  * - MDBM_REPLACE - Add if missing, replace if present
1118  * - MDBM_MODIFY - Fail if missing, replace if present
1119  *
1120  * NOTE: Ensure that the key and value datum pointers reference storage that
1121  * is external to the db and not the result of, for example, an
1122  * \ref mdbm_fetch, \ref mdbm_first, or \ref mdbm_next. A store operation can
1123  * result in various db internal modifications that invalidate previously
1124  * returned key and value pointers. Also be aware that mdbm_store_r
1125  * overwrites key.dptr and val.dptr, so to avoid memory leaks for the case
1126  * when key.dptr or val.dptr point to malloc-ed or new-ed memory, you should
1127  * keep a copy of above pointers and deallocate them when done.
1128  *
1129  * NOTE: If you do multiple stores to the same key using MDBM_REPLACE, the
1130  * corresponding value's location in memory can change after each store. The
1131  * MDBM_REPLACE flag does not imply that the value will reuse the same location.
1132  * Even if you store a smaller value for an existing key, it probably won't
1133  * reuse the existing value's location. In the current implementation, the only
1134  * time it tries to reuse a value's location is when replacing data with exactly
1135  * the same size. This internal optimization is subject to change in any future
1136  * release.
1137  *
1138  * \todo doc, example MDBM_RESERVE
1139  */
1140 extern int mdbm_store(MDBM *db, datum key, datum val, int flags);
1141 
1142 
1143 /**
1144  * Stores the record specified by the \a key and \a val parameters.
1145  *
1146  * \param[in,out] db Database handle
1147  * \param[in] key \ref datum to be used as the index key
1148  * \param[in] val \ref datum containing the value to be stored
1149  * \param[in] flags Type of store operation to be performed.
1150  * See \ref mdbm_store for flag values.
1151  * \param[in,out] iter MDBM iterator
1152  * \return Store status
1153  * \retval -1 Error, and errno is set
1154  * \retval 0 Success
1155  * \retval 1 Flag MDBM_INSERT was specified, and the key already exists
1156  *
1157  * See \ref mdbm_store for additional usage information.
1158  */
1159 extern int mdbm_store_r(MDBM *db, datum* key, datum* val, int flags, MDBM_ITER* iter);
1160 
1161 /**
1162  * Stores a string into the MDBM.
1163  *
1164  * \param[in,out] db Database handle
1165  * \param[in] key Key for storage
1166  * \param[in] val String to store
1167  * \param[in] flags Type of store operation to be performed.
1168  * \return Store status
1169  * \retval -1 Error, and errno is set
1170  * \retval 0 Success
1171  * \retval 1 Flag MDBM_INSERT was specified, and the key already exists
1172  *
1173  * See \ref mdbm_store for additional usage information.
1174  */
1175 extern int mdbm_store_str(MDBM *db, const char *key, const char *val, int flags);
1176 
1177 /** \} RecordAccessGroup */
1178 
1179 /**
1180  * \defgroup RecordIterationGroup Record Iteration Group
1181  * The Record Iteration Group contains the API for iterating over records in
1182  * an MDBM.
1183  * \{
1184  */
1185 
1186 /**
1187  * Returns the first key/value pair from the database. The order that records
1188  * are returned is not specified.
1189  *
1190  * \todo Is it safe to update/delete an entry returned via \ref mdbm_first?
1191  *
1192  * \param[in,out] db Database handle
1193  * \return kvpair. If database is empty, return a null kvpair (key and value dsize==0, dptr==NULL).
1194  */
1195 extern kvpair mdbm_first(MDBM *db);
1196 
1197 /**
1198  * Fetches the first record in an MDBM. Initializes the iterator, and returns
1199  * the first key/value pair from the db. Subsequent calls to \ref mdbm_next_r or
1200  * \ref mdbm_nextkey_r with this iterator will loop through the entire db.
1201  *
1202  * \param[in,out] db Database handle
1203  * \param[in,out] iter MDBM iterator
1204  * \return kvpair. If database is empty, return a null kvpair (key and value dsize==0, dptr==NULL).
1205  */
1206 extern kvpair mdbm_first_r(MDBM *db, MDBM_ITER *iter);
1207 
1208 /**
1209  * Returns the next key/value pair from the database. The order that records
1210  * are returned is not specified.
1211  *
1212  * \todo Is it safe to update/delete an entry returned via mdbm_next?
1213  *
1214  * \param[in,out] db Database handle
1215  * \return kvpair. If no more records exist,
1216  * return a null kvpair (key and value dsize==0, dptr==NULL).
1217  */
1218 extern kvpair mdbm_next(MDBM *db);
1219 
1220 /**
1221  * Fetches the next record in an MDBM. Returns the next key/value pair from
1222  * the db, based on the iterator.
1223  *
1224  * \param[in,out] db Database handle
1225  * \param[in,out] iter MDBM iterator
1226  * \return kvpair. If no more records exist,
1227  * return a null kvpair (key and value dsize==0, dptr==NULL).
1228  */
1229 extern kvpair mdbm_next_r(MDBM *db, MDBM_ITER *iter);
1230 
1231 /**
1232  * Returns the first key from the database. The order that records are
1233  * returned is not specified.
1234  *
1235  * \param[in,out] db Database handle
1236  * \return datum. If the database is empty, return a null datum (dsize==0, dptr==NULL).
1237  */
1238 extern datum mdbm_firstkey(MDBM *db);
1239 
1240 /**
1241  * Fetches the first key in an MDBM. Initializes the iterator, and returns
1242  * the first key from the db. Subsequent calls to \ref mdbm_next_r or
1243  * \ref mdbm_nextkey_r with this iterator will loop through the entire db.
1244  *
1245  * \param[in,out] db Database handle
1246  * \param[in,out] iter MDBM iterator
1247  * \return datum. If the database is empty, return a null datum (dsize==0, dptr==NULL).
1248  */
1249 extern datum mdbm_firstkey_r(MDBM *db, MDBM_ITER *iter);
1250 
1251 /**
1252  * Returns the next key pair from the database. The order that records are
1253  * returned is not specified.
1254  *
1255  * \param[in,out] db Database handle
1256  * \return datum. If the database is empty, return a null datum (dsize==0, dptr==NULL).
1257  */
1258 extern datum mdbm_nextkey(MDBM *db);
1259 
1260 /**
1261  * Fetches the next key in an MDBM. Returns the next key from the db.
1262  * Subsequent calls to \ref mdbm_next_r or mdbm_nextkey_r with this iterator
1263  * will loop through the entire db.
1264  *
1265  * \param[in,out] db Database handle
1266  * \param[in,out] iter MDBM iterator
1267  * \return datum. If there are no more records, return a null datum (dsize==0, dptr==NULL).
1268  */
1269 extern datum mdbm_nextkey_r(MDBM *db, MDBM_ITER *iter);
1270 
1271 #define MDBM_ENTRY_DELETED 0x1
1272 #define MDBM_ENTRY_LARGE_OBJECT 0x2
1273 
1274 typedef struct mdbm_page_info {
1275  uint32_t page_num;
1277  intptr_t page_addr;
1278  uint32_t page_size;
1299 
1300 typedef struct mdbm_entry_info {
1301  uint32_t entry_flags;
1302  uint32_t entry_index;
1303  uint32_t entry_offset;
1304  uint32_t key_offset;
1305  uint32_t val_offset;
1306  uint32_t lob_page_num;
1307  uint32_t lob_num_pages;
1308  intptr_t lob_page_addr;
1313 
1314 typedef struct mdbm_iterate_info {
1318 
1319 typedef int (*mdbm_iterate_func_t)(void* user, const mdbm_iterate_info_t* info, const kvpair* kv);
1320 
1321 #define MDBM_ITERATE_ENTRIES 0x01 /**< Iterate over page entries */
1322 #define MDBM_ITERATE_NOLOCK 0x80 /**< Iterate without locking */
1323 
1324 /**
1325  * Iterates through all keys starting on page \a pagenum and continuing
1326  * through the rest of the database. If \a flags contains
1327  * MDBM_ITERATE_ENTRIES, function \a func is invoked for each record. If
1328  * \a flag does not have MDBM_ITERATE_ENTRIES set, then \a func is invoke once
1329  * per page with \a kv set to NULL.
1330  *
1331  * \param[in,out] db Database handle
1332  * \param[in] pagenum Starting page number
1333  * \param[in] func Function to invoke for each key
1334  * \param[in] flags iteration control (below)
1335  * \param[in] user User-supplied opaque pointer to pass to \a func
1336  * \return Iteration status
1337  * \retval -1 Error
1338  * \retval 0 Success
1339  *
1340  * Values for \a flags mask:
1341  * - MDBM_ITERATE_NOLOCK - Do not lock when iterating
1342  * - MDBM_ITERATE_ENTRIES - Invoke \a func for each record
1343  */
1344 extern int mdbm_iterate(MDBM* db, int pagenum, mdbm_iterate_func_t func, int flags, void* user);
1345 
1346 /** \} RecordIterationGroup */
1347 
1348 
1349 /**
1350  * \defgroup LockingGroup Locking Group
1351  * The Locking Group contain the API for using the various lock types:
1352  * exclusive, partition, and shared.
1353  * \{
1354  */
1355 
1356 /**
1357  * Returns whether or not MDBM is locked by another process or thread (thread:
1358  * V3 only).
1359  *
1360  * \param[in,out] db Database handle
1361  * \return Locked status
1362  * \retval 0 Database is not locked
1363  * \retval 1 Database is locked
1364  */
1365 extern int mdbm_islocked(MDBM *db);
1366 
1367 /**
1368  * Returns whether or not MDBM is currently locked (owned) by the calling process.
1369  * \em Owned MDBMs have multiple nested locks in place.
1370  *
1371  * \param[in,out] db Database handle
1372  * \return Owned status
1373  * \retval 0 Database is not owned
1374  * \retval 1 Database is owned
1375  */
1376 extern int mdbm_isowned(MDBM *db);
1377 
1378 #define MDBM_LOCKMODE_UNKNOWN 0xFFFFFFFF /**< Returned by mdbm_get_lockmode for an unknown lock mode (MDBM V2). */
1379 
1380 /**
1381  * Gets the MDBM's lock mode.
1382  *
1383  * \param[in,out] db Database handle
1384  * \return Lock mode
1385  * \retval 0 - Exclusive locking
1386  * \retval MDBM_OPEN_NOLOCK - No locking
1387  * \retval MDBM_PARTITIONED_LOCKS - Partitioned locking
1388  * \retval MDBM_RW_LOCKS - Shared (read-write) locking
1389  */
1390 extern uint32_t mdbm_get_lockmode(MDBM *db);
1391 
1392 /**
1393  * Locks the database for exclusive access by the caller. The lock is
1394  * nestable, so a caller already holding the lock may call mdbm_lock again as
1395  * long as an equal number of calls to \ref mdbm_unlock are made to release the
1396  * lock.
1397  *
1398  * \param[in,out] db Database handle
1399  * \return Lock status
1400  * \retval -1 Error, and errno is set.
1401  * Typically, an error occurs if the database has been corrupted.
1402  * \retval 1 Success, exclusive lock was acquired
1403  */
1404 extern int mdbm_lock(MDBM *db);
1405 
1406 /**
1407  * Unlocks the database, releasing exclusive or shared access by the caller. If
1408  * the caller has called \ref mdbm_lock or \ref mdbm_lock_shared multiple times
1409  * in a row, an equal number of unlock calls are required. See \ref mdbm_lock
1410  * and \ref mdbm_lock_shared for usage.
1411  *
1412  * \param[in,out] db Database to be unlocked
1413  * \return Lock status
1414  * \retval -1 Error, and errno is set.
1415  * Typically, an error occurs if the database has been
1416  * corrupted, or the process/thread does not own the lock.
1417  * \retval 1 Success, exclusive lock was released
1418  */
1419 extern int mdbm_unlock(MDBM *db);
1420 
1421 /**
1422  * Attempts to exclusively lock an MDBM.
1423  *
1424  * \param[in,out] db Database handle
1425  * \return Lock status
1426  * \retval -1 Error, and errno is set
1427  * \retval 1 Success, exclusive lock was acquired
1428  */
1429 extern int mdbm_trylock(MDBM *db);
1430 
1431 /**
1432  * Locks a specific partition in the database for exclusive access by the
1433  * caller. The lock is nestable, so a caller already holding the lock may
1434  * call mdbm_plock again as long as an equal number of calls to \ref mdbm_punlock
1435  * are made to release the lock.
1436  *
1437  * \param[in,out] db Database to be locked.
1438  * \param[in] key Key to be hashed to determine page to lock
1439  * \param[in] flags Ignored.
1440  * \return Lock status
1441  * \retval -1 Error, and errno is set.
1442  * Typically, an error occurs if the database has been corrupted.
1443  * \retval 1 Success, partition lock was acquired
1444  */
1445 extern int mdbm_plock(MDBM *db, const datum *key, int flags);
1446 
1447 #define MDBM_LOCKMODE_UNKNOWN 0xFFFFFFFF /**< Returned by mdbm_get_lockmode for an unknown lock mode (MDBM V2). */
1448 
1449 /**
1450  * Unlocks a specific partition in the database, releasing exclusive access by
1451  * the caller. If the caller has called \ref mdbm_plock multiple times in a row,
1452  * an equal number of unlock calls are required. See \ref mdbm_plock for
1453  * usage.
1454  *
1455  * \param[in,out] db Database to be unlocked
1456  * \param[in] key Ignored.
1457  * \param[in] flags Ignored.
1458  * \return Lock status
1459  * \retval -1 Error, and errno is set.
1460  * Typically, an error occurs if the database has been corrupted,
1461  * or mdbm_plock is called without holding the lock
1462  * \retval 1 Success, partition lock was released
1463  */
1464 extern int mdbm_punlock(MDBM *db, const datum *key, int flags);
1465 
1466 /**
1467  * Tries to locks a specific partition in the database for exclusive access by
1468  * the caller. The lock is nestable, so a caller already holding the lock may
1469  * call \ref mdbm_plock again as long as an equal number of calls to \ref mdbm_punlock
1470  * are made to release the lock. See \ref mdbm_plock for usage.
1471  *
1472  * \param[in,out] db Database to be unlocked
1473  * \param[in] key Key to be hashed to determine page to lock
1474  * \param[in] flags Ignored.
1475  * \return Lock status
1476  * \retval -1 Error, and errno is set. Partition is not locked.
1477  * Typically, an error occurs if the database has been corrupted.
1478  * \retval 1 Success, partition lock was acquired
1479  */
1480 extern int mdbm_tryplock(MDBM *db, const datum *key, int flags);
1481 
1482 /**
1483  * Locks the database for shared access by readers, excluding access to
1484  * writers. This is multiple-readers, one writer (MROW) locking. The database
1485  * must be opened with the MDBM_RW_LOCKS flag to enable shared locks. Use
1486  * \ref mdbm_unlock to release a shared lock.
1487  *
1488  * Write access (ex. \ref mdbm_store and \ref mdbm_delete) must not be done in
1489  * the context of a shared lock. Write access must be done in the context of
1490  * an exclusive lock (\ref mdbm_lock and \ref mdbm_unlock).
1491  *
1492  * The lock is nestable, so a caller already holding the lock may call
1493  * mdbm_lock_shared again as long as an equal number of calls to \ref mdbm_unlock are
1494  * made to release the lock.
1495  *
1496  * \param[in,out] db Database handle
1497  * \return Lock status
1498  * \retval -1 Error, and errno is set.
1499  * Typically, an error occurs if the database has been corrupted.
1500  * \retval 1 Success, shared lock was acquired
1501  */
1502 extern int mdbm_lock_shared(MDBM *db);
1503 
1504 /**
1505  * Locks the database for shared access by readers, excluding access to
1506  * writers. This is the non-blocking version of \ref mdbm_lock_shared
1507  *
1508  * This is MROW locking. The database must be opened with the MDBM_RW_LOCKS
1509  * flag to enable shared locks.
1510  *
1511  * Write access (ex. \ref mdbm_store and \ref mdbm_delete) must not be done in the
1512  * context of a shared lock. Write access must be done in the context of an
1513  * exclusive lock (\ref mdbm_lock and \ref mdbm_unlock).
1514  *
1515  * \param[in,out] db Database handle
1516  * \return Lock status
1517  * \retval -1 Error, and errno is set.
1518  * Typically, an error occurs if the database has been corrupted,
1519  * or the database is already exclusively locked.
1520  * \retval 1 Success, shared lock was acquired
1521  */
1522 extern int mdbm_trylock_shared(MDBM *db);
1523 
1524 /**
1525  * Perform either partition, shared or exclusive locking based on the
1526  * locking-related flags supplied to \ref mdbm_open.
1527  *
1528  * \param[in,out] db Database handle
1529  * \param[in] key Key to be hashed to determine which page to lock (if needed)
1530  * \param[in] flags MDBM_O_RDWR means lock to perform a write
1531  * \return Lock status
1532  * \retval -1 Error, and errno is set.
1533  * Typically, an error occurs if the database has been corrupted.
1534  * \retval 1 Success, smart lock was acquired, or \a db was opened with MDBM_OPEN_NOLOCK.
1535  */
1536 extern int mdbm_lock_smart(MDBM *db, const datum *key, int flags);
1537 
1538 /**
1539  * Attempts to lock an MDBM based on the locking flags supplied to \ref mdbm_open.
1540  *
1541  * \param[in,out] db Database handle
1542  * \param[in] key Key to be hashed to determine page to lock (if needed)
1543  * \param[in] flags MDBM_O_RDWR means lock to perform a write.
1544  * \return Lock status
1545  * \retval -1 Error, and errno is set
1546  * \retval 1 Success, smart lock was acquired, or \a db was opened with MDBM_OPEN_NOLOCK.
1547  */
1548 extern int mdbm_trylock_smart(MDBM *db, const datum *key, int flags);
1549 
1550 /** Unlock an MDBM based on the locking flags supplied to \ref mdbm_open.
1551  *
1552  * \param[in,out] db Database handle
1553  * \param[in] key Key to be hashed to determine page to lock (if needed)
1554  * \param[in] flags Reserved for future use
1555  * \return Lock status
1556  * \retval -1 Error, and errno is set
1557  * \retval 1 Success, smart lock was released, or \a db was opened with MDBM_OPEN_NOLOCK.
1558  */
1559 extern int mdbm_unlock_smart(MDBM *db, const datum *key, int flags);
1560 
1561 /**
1562  * Resets the global lock ownership state of a database.
1563  * USE THIS FUNCTION WITH EXTREME CAUTION!
1564  * The global lock ownership state of an MDBM is visible to other
1565  * processes and is reset system-wide. Resetting a lock state while other
1566  * threads/processes are accessing this database might cause those operations
1567  * to fail.
1568  *
1569  * \param[in] dbfilename Database file name
1570  * \param[in] flags Reserved for future use, and must be 0.
1571  * \return Lock reset status
1572  * \retval -1 Error, and errno is set
1573  * \retval 0 Success
1574  */
1575 extern int mdbm_lock_reset(const char* dbfilename, int flags);
1576 
1577 /**
1578  * Removes all lockfiles associated with an MDBM file.
1579  * USE THIS FUNCTION WITH EXTREME CAUTION!
1580  * NOTE: This is only intended to clean-up lockfiles when removing an MDBM file.
1581  * This function can only be used when all processes that access
1582  * the MDBM whose locks are being deleted are not running.
1583  * Calling it on an MDBM still in use will cause corruption and undefined behavior.
1584  * Deleting lockfiles resets lock ownership and locking mode (exclusive/partition/shared).
1585  *
1586  * \param[in] dbname Full path of MDBM file
1587  * \return Lockfile deletion status
1588  * \retval -1 Error, and errno is set
1589  * \retval 0 Success
1590  */
1591 extern int mdbm_delete_lockfiles(const char* dbname);
1592 
1593 /** \} LockingGroup */
1594 
1595 /**
1596  * \defgroup DataManagementGroup Data Management Group
1597  * The Data Management Group contains the API managing data as a whole (not
1598  * record based), or on a large scale.
1599  * \{
1600  */
1601 
1602 #define MDBM_SAVE_COMPRESS_TREE 0x01000000 /**< Compress MDBM before saving */
1603 
1604 /**
1605  * Compresses the existing MDBM directory. Attempts to rebalance the directory
1606  * and to compress the db to a smaller size.
1607  *
1608  * NOTE: This function does not work with Windowed-Mode. For that use case, you should
1609  * export your data and re-import it into a clean MDBM.
1610  *
1611  * NOTE: Support for this feature in V4 should be considered experimental. Please run
1612  * it on an offline DB, after making a backup of your uncompressed DB.
1613  *
1614  * \param[in,out] db Database handle
1615  */
1616 extern void mdbm_compress_tree(MDBM *db);
1617 
1618 /**
1619  * Truncates the MDBM to single empty page
1620  *
1621  * V3 WARNING: This loses the existing configuration information (ex. large object support
1622  * or a hash function that was set other than the default).
1623  *
1624  * \param[in,out] db Database handle
1625  */
1626 extern void mdbm_truncate(MDBM *db);
1627 
1628 /**
1629  * Prunes an MDBM. Iterates through the database calling the supplied prune
1630  * function for each item in the database. Prune function:
1631  *
1632  * \code
1633  * int (*prune)(MDBM *, datum, datum, void *)
1634  * \endcode
1635  *
1636  * The user-supplied param pointer is passed as the 4th parameter to the prune
1637  * function. If the prune function returns 1, the item is deleted. If the
1638  * prune function returns 0, the item is retained.
1639  *
1640  * \param[in,out] db Database handle
1641  * \param[in] prune Prune function
1642  * \param[in] param User supplied param, passed to prune function.
1643  */
1644 extern void mdbm_prune(MDBM* db, int (*prune)(MDBM *, datum, datum, void *), void *param);
1645 
1646 /**
1647  * Purges (removes) all entries from an MDBM. This does not change the MDBM's
1648  * configuration or general structure.
1649  *
1650  * \param[in,out] db Database handle
1651  */
1652 extern void mdbm_purge(MDBM *db);
1653 
1655  void* user_data; /**< Opaque user data */
1656 };
1657 
1658 typedef int (*mdbm_clean_func)(MDBM *, const datum*, const datum*,
1659  struct mdbm_clean_data *, int* quit);
1660 
1661 /**
1662  * The specified cleaner function will be called by \ref mdbm_clean. It can
1663  * be called by \ref mdbm_store should space be needed on a page. \ref
1664  * mdbm_store will call it if there is no registered shake function (see \ref
1665  * mdbm_limit_size_v3) or the registered shake function does not clear enough
1666  * space for the store. An entry marked clean means it may be re-used to
1667  * store new data.
1668  *
1669  * NOTE: V3 API
1670  *
1671  * The clean function is typedef'd like the following:
1672  *
1673  * \code
1674  * typedef int (*mdbm_clean_func)(MDBM *, const datum*, const datum*,
1675  * struct mdbm_clean_data *,
1676  * int* quit);
1677  * \endcode
1678  *
1679  * When the mdbm_clean_func is called to check an entry on a page, it will be
1680  * passed the data parameter from the call to mdbm_set_cleanfunc.
1681  *
1682  * The quit parameter is used by callers of the mdbm_clean_func to determine
1683  * whether other entries on the page should be cleaned. If the mdbm_clean_func
1684  * sets quit to true (non 0) then no other entries on the page will be checked
1685  * for cleaning. Otherwise \ref mdbm_clean or \ref mdbm_store will continue checking
1686  * entries on the page to determine if they can be re-used/cleaned.
1687  *
1688  * The mdbm_clean_func should return true (non 0) if the key/value may be
1689  * re-used, AKA clean. IMPORTANT: To enable this feature cache mode must be
1690  * set on the database. Ex:
1691  * \code
1692  * mdbm_set_cachemode(dbh, MDBM_CACHEMODE_EVICT_CLEAN_FIRST|MDBM_CACHEMODE_GDSF);
1693  * mdbm_set_cleanfunc(dbh, mycleanerfunction, mycleanerdata);
1694  * mdbm_limit_size(dbh, limitNumPages, 0); // notice no shake function is specified
1695  * \endcode
1696  *
1697  * Both of the cache mode flags are required. The second
1698  * flag(MDBM_CACHEMODE_GDSF) used in the example can be replaced by
1699  * MDBM_CACHEMODE_LFU or MDBM_CACHEMODE_LRU.
1700  *
1701  * \param[in,out] db Database handle
1702  * \param[in] func User-provided function to determine re-use of an entry
1703  * \param[in] data Opaque data passed to the user-provided function \a func
1704  * \return Set clean function status
1705  * \retval -1 Error, and errno is set
1706  * \retval 0 Success
1707  */
1708 extern int mdbm_set_cleanfunc(MDBM* db, mdbm_clean_func func, void* data);
1709 
1710 /**
1711  * Mark entries clean/re-usable in the database for the specified page. If
1712  * pagenum is -1, then clean all pages. It relies on the user provided
1713  * callback function set via \ref mdbm_set_cleanfunc to determine
1714  * re-usability/cleanliness of an entry. To be clean means an entry can be
1715  * re-used to store new data.
1716  *
1717  * NOTE: V3 API
1718  *
1719  * \param[in,out] db Database handle
1720  * \param[in] pagenum Page number to start cleaning.
1721  * If < 0, then clean all pages in the database.
1722  * \param[in] flags Ignored
1723  * \return Clean status or count
1724  * \retval -1 Error
1725  * \retval Number of cleaned entries
1726  */
1727 extern int mdbm_clean(MDBM* db, int pagenum, int flags);
1728 
1729 /** \} DataManagementGroup */
1730 
1731 
1732 /**
1733  * \defgroup DataIntegrityGroup Data Integrity Group
1734  * The Data Integrity Group contain the API to check the state of internal
1735  * structures, or to control general data access.
1736  * \{
1737  */
1738 
1739 #define MDBM_CHECK_HEADER 0 /**< Check MDBM header for integrity */
1740 #define MDBM_CHECK_CHUNKS 1 /**< Check MDBM header and chunks (page structure) */
1741 #define MDBM_CHECK_DIRECTORY 2 /**< Check MDBM header, chunks, and directory */
1742 #define MDBM_CHECK_ALL 3 /**< Check MDBM header, chunks, directory, and data */
1743 
1744 /**
1745  * Checks an MDBM's integrity, and displays information on standard output.
1746  *
1747  * \param[in,out] db Database handle
1748  * \param[in] level Depth of checks
1749  * \param[in] verbose Whether to display verbose information while checking
1750  * \return Check status
1751  * \retval -1 Error
1752  * \retval 0 Success
1753  */
1754 extern int mdbm_check(MDBM* db, int level, int verbose);
1755 
1756 /**
1757  * Checks integrity of an entry on a page.
1758  *
1759  * NOTE: This has not been implemented.
1760  *
1761  * \param[in,out] db Database handle
1762  * \param[in] pagenum Page number
1763  * \param[in] mapped_pagenum Mapped page number
1764  * \param[in] index Entry number on a page
1765  */
1766 extern void mdbm_chk_error(MDBM* db, int pagenum, int mapped_pagenum, int index);
1767 
1768 /**
1769  * Checks the specified page for errors. It will print errors found on the
1770  * page, including bad key size, bad val size, and bad offsets of various
1771  * fields.
1772  *
1773  * V2: Prints found errors to stdout. If no errors, then no printing
1774  * performed. When it detects errors, it returns -1 and \ref mdbm_get_errno
1775  * returns EFAULT.
1776  *
1777  * V3: Prints found errors via mdbm_log targeting LOG_CRITICAL. If no
1778  * errors, then no logging performed. When it detects errors, it returns -1
1779  * and errno is set to EFAULT.
1780  *
1781  * \param[in,out] db Database handle
1782  * \param[in] pagenum Page to check for errors
1783  * \return Check page status
1784  * \retval -1 Errors detected, and errno is set.
1785  * This could be due to a locking error.
1786  * For v2, the caller must call \ref mdbm_get_errno and check against EFAULT.
1787  * For v3, the caller must check errno against EFAULT.
1788  * \retval 0 Success
1789  */
1790 extern int mdbm_chk_page(MDBM* db, int pagenum);
1791 
1792 /**
1793  * Checks the database for errors. It will report same as mdbm_chk_page for
1794  * all pages in the database. See v2 and v3 in \ref mdbm_chk_page to
1795  * determine if errors detected in the database.
1796  *
1797  * \param[in,out] db Database handle
1798  * \return Check all pages status
1799  * \retval -1 Errors detected, and errno is set.
1800  * This could be due to a locking error.
1801  * For v2, the caller must call \ref mdbm_get_errno and check against EFAULT.
1802  * For v3, the caller must check errno against EFAULT.
1803  * \retval 0 success
1804 
1805  */
1806 extern int mdbm_chk_all_page(MDBM* db);
1807 
1808 #define MDBM_PROT_NONE 0 /**< Page no access */
1809 #define MDBM_PROT_READ 1 /**< Page read access */
1810 #define MDBM_PROT_WRITE 2 /**< Page write access */
1811 
1812 #define MDBM_PROT_NOACCESS MDBM_PROT_NONE /**< Page no access */
1813 #define MDBM_PROT_ACCESS 4 /**< Page protection mask */
1814 
1815 /**
1816  * Sets all database pages to \a protect permission. This function is for
1817  * advanced users only. Users that want to use the built-in \em protect
1818  * feature should specify MDBM_PROTECT in their \ref mdbm_open flags.
1819  *
1820  * \param[in,out] db Database handle
1821  * \param[in] protect Permission mask
1822  * \return Protect status
1823  * \retval -1 Error, and errno is set
1824  * \retval 0 Success
1825  *
1826  * Values for \a protect mask:
1827  * - MDBM_PROT_NONE - no access
1828  * - MDBM_PROT_READ - read access
1829  * - MDBM_PROT_WRITE - write access
1830  * - MDBM_PROT_ACCESS - all access
1831  * - MDBM_PROT_NOACCESS - no access (same as MDBM_PROT_NONE)
1832  *
1833  * NOTE: RHEL is unable to set MDBM_PROT_WRITE without MDBM_PROT_READ, so specifying
1834  * MDBM_PROT_WRITE does not protect against reads.
1835  */
1836 extern int mdbm_protect(MDBM* db, int protect);
1837 
1838 /** \} DataIntegrityGroup */
1839 
1840 
1841 /**
1842  * \defgroup DataDisplayGroup Data Display Group
1843  * The Data Display Group contains the API to display page data.
1844  * These routines are intended for debugging purposes
1845  * \{
1846  */
1847 
1848 /**
1849  * Dumps information for all pages, in version-specific format, to standard output.
1850  *
1851  * \param[in,out] db Database handle
1852  */
1853 extern void mdbm_dump_all_page(MDBM *db);
1854 
1855 /**
1856  * Dumps specified page's information, in version-specific format, to standard output.
1857  *
1858  * \param[in,out] db Database handle
1859  * \param[in] pno Page number
1860  */
1861 extern void mdbm_dump_page(MDBM *db, int pno);
1862 
1863 /** \} DataDisplayGroup */
1864 
1865 /**
1866  * \defgroup StatisticsGroup Statistics Group
1867  * The Statistics Group contains the API for capacity and performance indicators.
1868  * \{
1869  */
1870 
1871 
1872 typedef struct mdbm_stats {
1891  uint32_t s_cache_mode;
1892 } mdbm_stats_t;
1893 
1894 
1895 typedef uint64_t mdbm_counter_t;
1896 
1897 typedef enum {
1898  MDBM_STAT_TYPE_FETCH = 0, /**< fetch* operations */
1899  MDBM_STAT_TYPE_STORE = 1, /**< store* operations */
1900  MDBM_STAT_TYPE_DELETE = 2, /**< delete* operations */
1902 } mdbm_stat_type;
1903 
1904 /**
1905  * Gets the number of operations performed for a stat \a type.
1906  *
1907  * \param[in,out] db Database handle
1908  * \param[in] type Stat type to return
1909  * \param[out] value Pointer to returned stat's value
1910  * \return Stat counter status
1911  * \retval -1 Error
1912  * \retval 0 Success
1913  *
1914  * Values for \a type:
1915  * - MDBM_STAT_TYPE_FETCH - For fetch* operations
1916  * - MDBM_STAT_TYPE_STORE - For store* operations
1917  * - MDBM_STAT_TYPE_DELETE - For delete* operations
1918  *
1919  * Stat operations must be enabled for operations to be tracked.
1920  * Use \ref mdbm_enable_stat_operations to enable this feature.
1921  *
1922  * Once enabled, statistics are persisted in the MDBM
1923  * and are not reset on \ref mdbm_close.
1924  *
1925  * Use program `mdbm_stat -H' to display stat operation metrics stored in the header.
1926  */
1927 extern int mdbm_get_stat_counter(MDBM *db, mdbm_stat_type type, mdbm_counter_t *value);
1928 
1929 /**
1930  * Gets the last time when an \a type operation was performed.
1931  *
1932  * \param[in,out] db Database handle
1933  * \param[in] type Stat type to return.
1934  * \param[out] value Pointer to returned stat's value.
1935  * \return Stat time status
1936  * \retval -1 Error
1937  * \retval 0 Success
1938  *
1939  * Values for \a type:
1940  * - MDBM_STAT_TYPE_FETCH - For fetch* operations
1941  * - MDBM_STAT_TYPE_STORE - For store* operations
1942  * - MDBM_STAT_TYPE_DELETE - For delete* operations
1943  *
1944  * Stat operations must be enabled for operations to be tracked.
1945  * Use \ref mdbm_enable_stat_operations to enable this feature.
1946  *
1947  * Once enabled, statistics are persisted in the MDBM
1948  * and are not reset on \ref mdbm_close.
1949  *
1950  * Use program `mdbm_stat -H' to display stat operation metrics stored in the header.
1951  */
1952 extern int mdbm_get_stat_time(MDBM *db, mdbm_stat_type type, time_t *value);
1953 
1954 /**
1955  * Resets the stat counter and last-time performed for fetch, store, and remove operations.
1956  *
1957  * \param[in,out] db Database handle
1958  *
1959  * Stat operations must be enabled for operations to be tracked.
1960  * Use \ref mdbm_enable_stat_operations to enable this feature.
1961  * If stat operations are not enabled, using this function will merely reset and
1962  * already cleared storage.
1963  *
1964  * Use program `mdbm_stat -H' to display stat operation metrics stored in the header.
1965  */
1966 extern void mdbm_reset_stat_operations(MDBM *db);
1967 
1968 /**
1969  * Enables and disables gathering of stat counters and/or last-time performed
1970  * for fetch, store, and remove operations.
1971  *
1972  * \param[in,out] db Database handle
1973  * \param[in] flags
1974  *
1975  * Enables stat operations so that we can track one or both of:
1976  * 1. Operations counters fetch, store and remove.
1977  * 2. Last timestamp when a fetch, store or delete was performed.
1978  *
1979  * flags = MDBM_STATS_BASIC enables gathering only the stats counters.
1980  * flags = MDBM_STATS_TIMED enables gathering only the stats timestamps.
1981  * flags = (MDBM_STATS_BASIC | MDBM_STATS_TIMED) enables both the stats counters and timestamps.
1982  * flags = 0 disables gathering of stats counters and timestamps.
1983  *
1984  * \retval -1 Error
1985  * \retval 0 Success
1986 
1987  * Use program `mdbm_stat -H' to display stat operation metrics stored in the header.
1988  */
1989 extern int mdbm_enable_stat_operations(MDBM *db, int flags);
1990 
1991 
1992 #define MDBM_CLOCK_STANDARD 0
1993 #define MDBM_CLOCK_TSC 1
1994 
1995 /**
1996  * Tells the MDBM library whether to use TSC (CPU TimeStamp Counters)
1997  * for timing the performance of fetch, store and delete operations.
1998  * The standard behavior of timed stat operations is to use clock_gettime(MONOTONIC)
1999  *
2000  * \param[in,out] db Database handle
2001  * \param[in] flags
2002  *
2003  * flags == MDBM_CLOCK_TSC Enables use of TSC
2004  * flags == MDBM_CLOCK_STANDARD Disables use of TSC
2005  *
2006  * \retval -1 Error
2007  * \retval 0 Success
2008  */
2009 extern int mdbm_set_stat_time_func(MDBM *db, int flags);
2010 
2011 
2012 /* mdbm_set_stats_func flags */
2013 #define MDBM_STATS_BASIC 0x1 /**< Basic stats only */
2014 #define MDBM_STATS_TIMED 0x2 /**< SLOW! Stats that call get-time functions. */
2015 
2016 /* Stat Callback Flags */
2017 #define MDBM_STAT_CB_INC 0x0 /**< Basic stat. Value is a delta. */
2018 #define MDBM_STAT_CB_SET 0x1 /**< Basic stat. Value is the current value. */
2019 #define MDBM_STAT_CB_ELAPSED 0x2 /**< Value is a time delta. */
2020 #define MDBM_STAT_CB_TIME 0x3 /**< Value is the stat time. */
2021 
2022 /* Macro for creating a unique integer tag for all stats metrics */
2023 #define COMBINE_STAT_TAG(tag, flag) (tag | (flag<<16))
2024 
2025 /**
2026  * The various stats metrics (tags):
2027  * All time/latency related stats (defined below) require:
2028  * mdbm_set_stats_func(db, flags|MDBM_STAT_CB_TIME, ...)
2029  */
2030 
2031 #define MDBM_STAT_TAG_FETCH 1 /* Successful fetch stats-callback counter */
2032 #define MDBM_STAT_TAG_STORE 2 /* Successful store stats-callback counter */
2033 #define MDBM_STAT_TAG_DELETE 3 /* Successful delete stats-callback counter */
2034 #define MDBM_STAT_TAG_LOCK 4 /* lock stats-callback counter (not implemented) */
2035 #define MDBM_STAT_TAG_FETCH_UNCACHED 5 /* Cache-miss with cache+backingstore */
2036 #define MDBM_STAT_TAG_GETPAGE 6 /* Generic access counter in windowed mode */
2037 #define MDBM_STAT_TAG_GETPAGE_UNCACHED 7 /* Windowed-mode "miss" (load new page into window) */
2038 #define MDBM_STAT_TAG_CACHE_EVICT 8 /* Cache evict stats-callback counter */
2039 #define MDBM_STAT_TAG_CACHE_STORE 9 /* Successful cache store counter (BS only) */
2040 #define MDBM_STAT_TAG_PAGE_STORE 10 /* Successful page-level store indicator */
2041 #define MDBM_STAT_TAG_PAGE_DELETE 11 /* Successful page-level delete indicator */
2042 #define MDBM_STAT_TAG_SYNC 12 /* Counter of mdbm_syncs and fsyncs */
2043 #define MDBM_STAT_TAG_FETCH_NOT_FOUND 13 /* Fetch cannot find a key in MDBM */
2044 #define MDBM_STAT_TAG_FETCH_ERROR 14 /* Error occurred during fetch */
2045 #define MDBM_STAT_TAG_STORE_ERROR 15 /* Error occurred during store (e.g. MODIFY failed) */
2046 #define MDBM_STAT_TAG_DELETE_FAILED 16 /* Delete failed: cannot find a key in MDBM */
2047 
2048 /** Fetch latency (expensive to collect)
2049  */
2050 #define MDBM_STAT_TAG_FETCH_LATENCY COMBINE_STAT_TAG(MDBM_STAT_TAG_FETCH, MDBM_STAT_CB_ELAPSED)
2051 /** Store latency (expensive to collect)
2052  */
2053 #define MDBM_STAT_TAG_STORE_LATENCY COMBINE_STAT_TAG(MDBM_STAT_TAG_STORE, MDBM_STAT_CB_ELAPSED)
2054 /** Delete latency (expensive to collect)
2055  */
2056 #define MDBM_STAT_TAG_DELETE_LATENCY COMBINE_STAT_TAG(MDBM_STAT_TAG_DELETE, MDBM_STAT_CB_ELAPSED)
2057 /** timestamp of last fetch (not yet implemented)
2058  */
2059 #define MDBM_STAT_TAG_FETCH_TIME COMBINE_STAT_TAG(MDBM_STAT_TAG_FETCH, MDBM_STAT_CB_TIME)
2060 /** timestamp of last store (not yet implemented)
2061  */
2062 #define MDBM_STAT_TAG_STORE_TIME COMBINE_STAT_TAG(MDBM_STAT_TAG_STORE, MDBM_STAT_CB_TIME)
2063 /** timestamp of last delete (not yet implemented)
2064  */
2065 #define MDBM_STAT_TAG_DELETE_TIME COMBINE_STAT_TAG(MDBM_STAT_TAG_DELETE, MDBM_STAT_CB_TIME)
2066 /** Cache miss latency for cache+Backingstore only (expensive to collect)
2067  */
2068 #define MDBM_STAT_TAG_FETCH_UNCACHED_LATENCY \
2069  COMBINE_STAT_TAG(MDBM_STAT_TAG_FETCH_UNCACHED, MDBM_STAT_CB_ELAPSED)
2070 /** access latency in windowed mode (expensive to collect)
2071  */
2072 #define MDBM_STAT_TAG_GETPAGE_LATENCY COMBINE_STAT_TAG(MDBM_STAT_TAG_GETPAGE, MDBM_STAT_CB_ELAPSED)
2073 /** windowed-mode miss latency (expensive to collect)
2074  */
2075 #define MDBM_STAT_TAG_GETPAGE_UNCACHED_LATENCY \
2076  COMBINE_STAT_TAG(MDBM_STAT_TAG_GETPAGE_UNCACHED, MDBM_STAT_CB_ELAPSED)
2077 /** cache evict latency (expensive to collect)
2078  */
2079 #define MDBM_STAT_TAG_CACHE_EVICT_LATENCY \
2080  COMBINE_STAT_TAG(MDBM_STAT_TAG_CACHE_EVICT, MDBM_STAT_CB_ELAPSED)
2081 /** Cache store latency in Cache+backingstore mode only (expensive to collect)
2082  */
2083 #define MDBM_STAT_TAG_CACHE_STORE_LATENCY \
2084  COMBINE_STAT_TAG(MDBM_STAT_TAG_CACHE_STORE, MDBM_STAT_CB_ELAPSED)
2085 /** Indicates a store occurred on a particular page. Value returned by callback is the page
2086  * number. It is up to the callback function to maintain a per-page count
2087  */
2088 #define MDBM_STAT_TAG_PAGE_STORE_VALUE COMBINE_STAT_TAG(MDBM_STAT_TAG_PAGE_STORE, MDBM_STAT_CB_SET)
2089 /** Indicates a delete occurred on a particular page. Value returned by callback is the page
2090  * number. It is up to the callback function to maintain a per-page count
2091  */
2092 #define MDBM_STAT_TAG_PAGE_DELETE_VALUE \
2093  COMBINE_STAT_TAG(MDBM_STAT_TAG_PAGE_DELETE, MDBM_STAT_CB_SET)
2094 /** mdbm_sync/fsync latency (expensive to collect)
2095  */
2096 #define MDBM_STAT_TAG_SYNC_LATENCY COMBINE_STAT_TAG(MDBM_STAT_TAG_SYNC, MDBM_STAT_CB_ELAPSED)
2097 
2098 
2099 /**
2100  * The general stats callback function definition.
2101  *
2102  * \param[in,out] db Database handle
2103  * \param[in] tag Integer identifier of the stat.
2104  * Use \ref mdbm_get_stat_name to get the string id.
2105  * \param[in] flags A MDBM_STAT_CB_* value indicating the kind of value.
2106  * \param[in] value Update to the indicated value
2107  * \param[in] user User-defined callback for data passed into \ref mdbm_set_stats_func
2108  */
2109 typedef void (*mdbm_stat_cb)(MDBM* db, int tag, int flags, uint64_t value, void* user);
2110 
2111 /**
2112  * Gets the name of a stat.
2113  *
2114  * \param[in] tag Stat identifier
2115  * \return Name of stat identifier. "unknown_tag" is return if \a tag is unknown.
2116  *
2117  */
2118 extern const char* mdbm_get_stat_name(int tag);
2119 
2120 /**
2121  * Sets the callback function for a stat.
2122  *
2123  * \param[in,out] db Database handle
2124  * \param[in] flags Passed to callback function
2125  * \param[in] cb Callback function
2126  * \param[in] user Opaque user information
2127  * \return Set stats function status
2128  * \retval -1 Error, and errno is set
2129  * \retval 0 Success, and callback is set
2130  */
2131 extern int mdbm_set_stats_func(MDBM* db, int flags, mdbm_stat_cb cb, void* user);
2132 
2133 /**
2134  * Prints to stdout various pieces of information, specifically: Total mapped
2135  * db size, Total number of entries, ADDRESS SPACE page efficiency, ADDRESS
2136  * SPACE byte efficiency, PHYSICAL MEM/DISK SPACE efficiency, Average bytes
2137  * per record, Maximum B-tree level, Minimum B-tree level, Minimum free bytes
2138  * on page, Minimum free bytes on page post compress
2139  *
2140  * If there are large objects, it loops through and reports record entries per page.
2141  *
2142  * NOTE: There is only a V2 implementation. V3 not currently supported.
2143  *
2144  * \param[in,out] db Database handle
2145  */
2146 extern void mdbm_stat_all_page(MDBM *db);
2147 
2148 /**
2149  * Gets a a stats block with individual stat values.
2150  *
2151  * \param[in,out] db Database handle
2152  * \param[out] s Stats block
2153  * \param[in] stats_size Stats block \a s size. Only as many stats will
2154  * be returned as according to this size. The stats block is
2155  * filled from top-to-bottom.
2156  * \return Get stats status
2157  * \retval -1 Error, and errno is set
2158  * \retval 0 Success
2159  */
2160 extern int mdbm_get_stats(MDBM* db, mdbm_stats_t* s, size_t stats_size);
2161 
2162 typedef struct mdbm_db_info {
2163  uint32_t db_page_size;
2164  uint32_t db_num_pages;
2165  uint32_t db_max_pages;
2167  uint32_t db_dir_width;
2172  uint32_t db_hash_func;
2173  const char* db_hash_funcname;
2174  uint32_t db_spill_size;
2175  uint32_t db_cache_mode;
2176 } mdbm_db_info_t;
2177 
2178 /**
2179  * Gets configuration information about a database.
2180  *
2181  * \param[in,out] db Database handle
2182  * \param[out] info Database information
2183  * \return Database info status
2184  * \retval -1 Error, and errno is set
2185  * \retval 0 Success
2186  */
2187 extern int mdbm_get_db_info(MDBM* db, mdbm_db_info_t* info);
2188 
2189 enum {
2190  MDBM_PTYPE_FREE = 0, /**< Page type free */
2191  MDBM_PTYPE_DATA = 1, /**< Page type data */
2192  MDBM_PTYPE_DIR = 2, /**< Page type directory */
2193  MDBM_PTYPE_LOB = 3 /**< Page type large object */
2194 };
2195 
2196 typedef struct mdbm_chunk_info {
2197  uint32_t page_num;
2198  uint32_t page_type;
2199  uint32_t dir_page_num;
2200  uint32_t num_pages;
2201  uint32_t page_data;
2202  uint32_t page_size;
2203  uint32_t prev_num_pages;
2204  uint32_t num_entries;
2205  uint32_t used_space;
2207 
2208 typedef int (*mdbm_chunk_iterate_func_t)(void* user, const mdbm_chunk_info_t* info);
2209 
2210 /**
2211  * Iterates over all pages (starting at page 0). This function only does
2212  * per-page iteration, where mdbm_iterate is intended for per-record iteration
2213  * starting at a page number.
2214  *
2215  * \param[in,out] db Database handle
2216  * \param[in] func Function to invoke for each chunk
2217  * \param[in] flags Iterations flags (below)
2218  * \param[in] user Opaque user pointer passed to \a func
2219  * \return Chunk iterate status
2220  * \retval -1 Error, and errno is set
2221  * \retval 0 Success
2222  *
2223  * Values for \a flags mask:
2224  * - MDBM_ITERATE_NOLOCK - Do not lock during iteration
2225  */
2226 extern int mdbm_chunk_iterate(MDBM* db, mdbm_chunk_iterate_func_t func, int flags, void* user);
2227 
2228 #define MDBM_STAT_DELETED 0x1 /**< Deprecated */
2229 #define MDBM_STAT_KEYS 0x2 /**< Deprecated */
2230 #define MDBM_STAT_VALUES 0x4 /**< Deprecated */
2231 #define MDBM_STAT_PAGES_ONLY 0x8 /**< Deprecated */
2232 #define MDBM_STAT_NOLOCK 0x80 /**< Do not lock for stat operation */
2233 
2234 #define MDBM_STAT_BUCKETS 20 /**< Number of buckets for stat histograms */
2235 
2236 typedef struct mdbm_bucket_stat {
2237  uint32_t num_pages;
2238  uint32_t min_bytes;
2239  uint32_t max_bytes;
2240  uint32_t min_free_bytes;
2241  uint32_t max_free_bytes;
2242  uint64_t sum_entries;
2243  uint64_t sum_bytes;
2244  uint64_t sum_free_bytes;
2246 
2247 typedef struct mdbm_stat_info {
2248  int flags;
2249 
2252  uint64_t sum_key_bytes;
2256 
2259  uint32_t min_key_bytes;
2260  uint32_t max_key_bytes;
2261  uint32_t min_val_bytes;
2262  uint32_t max_val_bytes;
2263  uint32_t min_lob_bytes;
2264  uint32_t max_lob_bytes;
2266 
2267  uint32_t max_data_pages;
2268  uint32_t num_free_pages;
2272  uint32_t num_lob_pages;
2273 
2275 
2279 
2280 
2281 /**
2282  * Gets overall database stats.
2283  *
2284  * \param[in,out] db Database handle
2285  * \param[out] info Database page-based stats
2286  * \param[out] stats Database record-based stats
2287  * \param[in] flags Iteration control (below)
2288  * \return Database stats status
2289  * \retval -1 Error, errno is set
2290  * \retval 0 Success
2291  *
2292  * Values for \a flags mask:
2293  * - MDBM_STAT_NOLOCK - Do not lock for overall operation
2294  * - MDBM_ITERATE_NOLOCK - Do no lock for page-based iteration
2295  */
2296 extern int mdbm_get_db_stats(MDBM* db, mdbm_db_info_t* info, mdbm_stat_info_t* stats, int flags);
2297 
2298 typedef struct mdbm_window_stats {
2299  uint64_t w_num_reused;
2300  uint64_t w_num_remapped;
2301  uint32_t w_window_size;
2304 
2305 /**
2306  * Used to retrieve statistics about windowing usage on the associated database.
2307  *
2308  * The V3 statistics structure is as follows. It may be extended in later versions, in which
2309  * case the s_size parameter should be set to the sizeof the struct used for that version.
2310  * \code
2311  * typedef struct mdbm_window_stats {
2312  * uint64_t w_num_reused;
2313  * uint64_t w_num_remapped;
2314  * uint32_t w_window_size;
2315  * uint32_t w_max_window_used;
2316  * } mdbm_window_stats_t;
2317  * \endcode
2318  *
2319  * NOTE: V3 API
2320  *
2321  * \param[in,out] db Database handle
2322  * \param[out] stats Pointer to struct that will receive the statistics
2323  * \param[in] s_size Size of \a stats.
2324  * Should be sizeof(mdbm_window_stats_t) for V3.
2325  * \return Get window stats status
2326  * \retval -1 Error
2327  * \retval 0 Success
2328  */
2329 extern int mdbm_get_window_stats(MDBM* db, mdbm_window_stats_t* stats, size_t s_size);
2330 
2331 /**
2332  * Counts the number of records in an MDBM.
2333  *
2334  * \param[in] db Database handle.
2335  * \return Number of records stored in an MDBM (use instead of count_all_page).
2336  */
2337 extern uint64_t mdbm_count_records(MDBM *db);
2338 
2339 /**
2340  * Counts the number of pages used by an MDBM.
2341  *
2342  * \param[in] db Database handle.
2343  * \return Number of pages: Count of all directory+data+LargeObject pages used by an MDBM
2344  */
2345 extern mdbm_ubig_t mdbm_count_pages(MDBM *db);
2346 
2347 
2348 /** \} StatisticsGroup */
2349 
2350 /**
2351  * \defgroup CacheAndBackingStoreGroup Cache and Backing Store Group
2352  * The Cache and Backing Store Group contain the API for configuring caches
2353  * and backing stores.
2354  * \{
2355  */
2356 
2357 #define MDBM_CACHEMODE_NONE 0 /**< No caching behavior */
2358 #define MDBM_CACHEMODE_LFU 1 /**< Entry with smallest number of accesses is evicted */
2359 #define MDBM_CACHEMODE_LRU 2 /**< Entry with oldest access time is evicted */
2360 #define MDBM_CACHEMODE_GDSF 3 /**< Greedy dual-size frequency (size and frequency) eviction */
2361 #define MDBM_CACHEMODE_MAX 3 /**< Maximum cache mode value */
2362 
2363 #define MDBM_CACHEMODE_EVICT_CLEAN_FIRST 0x10 /**< add to cachemode to evict clean items 1st */
2364 /** Extracts cache mode */
2365 #define MDBM_CACHEMODE(m) ((m)&3)
2366 /** Defines a mask for the cache mode, including control (eviction) bit. */
2367 #define MDBM_CACHEMODE_BITS (3 | MDBM_CACHEMODE_EVICT_CLEAN_FIRST)
2368 
2369 /**
2370  * Manage the database as a cache. mdbm_set_cachemode must be called before
2371  * data is inserted. Tracking metadata is stored with each entry which allows
2372  * MDBM to do cache eviction via LRU, LFU, and GDSF
2373  * (greedy-dual-size-frequency). MDBM also supports clean/dirty tracking and
2374  * the application can supply a callback (see \ref mdbm_set_backingstore)
2375  * which is called by MDBM when a dirty entry is about to be evicted allowing
2376  * the application to sync the entry to a backing store or perform some other
2377  * type of "clean" operation.
2378  *
2379  * NOTE: V3 API
2380  *
2381  * \param[in,out] db Database handle
2382  * \param[in] cachemode caching mode value (below)
2383  * \return Set cache mode status
2384  * \retval -1 Error, with errno set.
2385  * - ENOSYS: wrong MDBM version
2386  * - EINVAL: bad \a cachemode value, or empty MDBM
2387  * - other: for locking errors
2388  * \retval 0 Success
2389  *
2390  * Values for \a cachemode:
2391  * - MDBM_CACHEMODE_NONE - no cache mode
2392  * - MDBM_CACHEMODE_LFU - least frequently used
2393  * - MDBM_CACHEMODE_LRU - least recently used
2394  * - MDBM_CACHEMODE_GDSF - greedy dual-size frequency
2395  */
2396 extern int mdbm_set_cachemode(MDBM* db, int cachemode);
2397 
2398 /**
2399  * Returns the current cache style of the database. See the cachemode parameter in
2400  * \ref mdbm_set_cachemode for the valid values.
2401  *
2402  * NOTE: V3 API
2403  *
2404  * \param[in,out] db Database handle
2405  * \return Current cache mode
2406  * \retval -1 error, with errno set.
2407  * - ENOSYS: wrong MDBM version
2408  * \retval MDBM_CACHEMODE_NONE
2409  * \retval MDBM_CACHEMODE_LFU
2410  * \retval MDBM_CACHEMODE_LRU
2411  * \retval MDBM_CACHEMODE_GDSF
2412  */
2413 extern int mdbm_get_cachemode(MDBM* db);
2414 
2415 /**
2416  * Returns the cache mode as a string. See \ref mdbm_set_cachemode
2417  *
2418  * NOTE: V3 API
2419  *
2420  * \param[in,out] cachemode Cache mode type (below)
2421  * \return Printable string representing a valid cache mode, otherwise the string "unknown"
2422  *
2423  * Values for \a cachemode:
2424  * - MDBM_CACHEMODE_NONE - no cache mode
2425  * - MDBM_CACHEMODE_LFU - least frequently used
2426  * - MDBM_CACHEMODE_LRU - least recently used
2427  * - MDBM_CACHEMODE_GDSF - greedy dual-size frequency
2428  */
2429 extern const char* mdbm_get_cachemode_name(int cachemode);
2430 
2431 typedef struct mdbm_bsops {
2432  void* (*bs_init)(MDBM* db, const char* filename, void* opt, int flags);
2433  int (*bs_term)(void* data, int flags);
2434  int (*bs_lock)(void* data, const datum* key, int flags);
2435  int (*bs_unlock)(void* data, const datum* key);
2436  int (*bs_fetch)(void* data, const datum* key, datum* val, datum* buf, int flags);
2437  int (*bs_store)(void* data, const datum* key, const datum* val, int flags);
2438  int (*bs_delete)(void* data, const datum* key, int flags);
2439  void* (*bs_dup)(MDBM* db, MDBM* newdb, void* data);
2440 } mdbm_bsops_t;
2441 
2442 #define MDBM_BSOPS_FILE ((const mdbm_bsops_t*)-1) /**< Backing store File identifier */
2443 #define MDBM_BSOPS_MDBM ((const mdbm_bsops_t*)-2) /**< Backing store MDBM identifier */
2444 
2445 /**
2446  * The backing-store support controls the in-memory cache as a read-through,
2447  * write-through cache. The backing-store interface uses a plugin model where
2448  * each plugin supplies lock, unlock, fetch, store, and delete functions. Two
2449  * predefined plugins are available: file-based(MDBM_BSOPS_FILE) and
2450  * MDBM-based (MDBM_BSOPS_MDBM). Typically, a window will be set to access the
2451  * backing store via \ref mdbm_set_window_size since the backing store may be
2452  * very large. Please refer to \ref mdbm_set_window_size for window size
2453  * restrictions.
2454  *
2455  * An \ref mdbm_store on \a cachedb must be used to update data in the backing
2456  * store. An in-place update (fetching the data address using \ref mdbm_fetch and
2457  * directly modifying the data within the mapped db) will not get written
2458  * through to the backing store.
2459  *
2460  * Once backing store is set on the opened cache, it cannot be removed from
2461  * the open cache.
2462  *
2463  * If you use MDBM_BSOPS_MDBM to set an MDBM backing store, the \a cachedb
2464  * will \e own the MDBM handle of the backing store, and \a cachedb will close
2465  * it when the cache MDBM handle is closed. You should not make any MDBM
2466  * calls directly using the backing store MDBM handle after passing it to
2467  * mdbm_set_backingstore.
2468  *
2469  * NOTE: V3 API
2470  *
2471  * \param[in,out] cachedb Database handle (for the cache database)
2472  * \param[in] bsops Can be MDBM_BSOPS_FILE, MDBM_BSOPS_MDBM, or user-supplied functions
2473  * \param[in] opt Depends on the bsops parameter. (ex., for MDBM_BSOPS_FILE
2474  * \a opt should be a file path, or for MDBM_BSOPS_MDBM \a opt should be an
2475  * MDBM database handle (opened with MDBM_OPEN_WINDOWED flag))
2476  * \param[in] flags Depends on the \a bsops.
2477  * Specify 0 if \a bsops is MDBM_BSOPS_MDBM.
2478  * Specify O_DIRECT to prevent paging out important data
2479  * when the bsops is MDBM_BSOPS_FILE.
2480  * \return Set backingstore status
2481  * \retval -1 Error
2482  * \retval 0 Success
2483  */
2484 extern int mdbm_set_backingstore(MDBM* cachedb, const mdbm_bsops_t* bsops, void* opt, int flags);
2485 
2486 /** \} CacheAndBackingStoreGroup */
2487 
2488 
2489 /**
2490  * \defgroup ImportExportGroup Import and Export Group
2491  * The Import and Export Group contains the APIs for reading and writing records to a file in
2492  * cdb_dump or db_dump format.
2493  *
2494  * Utility functions for converting from MDBM files into a textual representation.
2495  * These functions generate data in one of 2 formats:
2496  * - The "printable" BerkeleyDB db_{dump,load} format
2497  * - The cdb_dump format, which is a little more compact because it uses binary
2498  * representation instead of escaped hex sequences: http://cr.yp.to/cdb/cdbmake.html
2499  * \{
2500  */
2501 
2502 /**
2503  * Export API: Use cdb_dump format to write a key/value pair to file opened
2504  * with fopen.
2505  *
2506  * \param[in] kv Key+Value pair
2507  * \param[in,out] fp FILE pointer (return value of fopen)
2508  * \return Dump kvpair to file status
2509  * \retval -1 Error, and errno is set
2510  * \retval 0 Success
2511  */
2512 extern int mdbm_cdbdump_to_file(kvpair kv, FILE *fp);
2513 
2514 /**
2515  * Export API: Finalize and close a file written in cdb_dump format and opened
2516  * with fopen. Adds newline character as the file "trailer" and closes the
2517  * file.
2518  *
2519  * \param[in,out] fp FILE pointer (return value of fopen)
2520  * \return Dump trailer to file status
2521  * \retval -1 Error, and errno is set
2522  * \retval 0 Success
2523  */
2524 extern int mdbm_cdbdump_trailer_and_close(FILE *fp);
2525 
2526 /**
2527  * Export API: Use cdb_dump format to write a key/value pair to a malloc'd buffer
2528  *
2529  * \param[in] kv Key+Value pair
2530  * \param[in,out] datasize Size of cdb_dump data to be added added to \a bufptr
2531  * \param[in,out] bufptr malloc'd buffer. bufptr may be realloc'd if \a
2532  * datasize is greater than \a bufsize
2533  * \param[in] bufsize Size of malloc'd buffer
2534  * \param[in] buf_offset Offset in \a bufptr to start writing data
2535  * \return Add record status
2536  * \retval -1 Error, and errno is set
2537  * \retval 0 Success
2538  */
2539 extern int mdbm_cdbdump_add_record(kvpair kv, uint32_t *datasize, char **bufptr,
2540  uint32_t *bufsize, uint32_t buf_offset);
2541 
2542 
2543 /**
2544  * Export API: Use db_dump format to write a key/value pair to a file opened with fopen.
2545  *
2546  * \param[in] kv Key+Value pair
2547  * \param[in,out] fp FILE pointer (return value of fopen)
2548  * \return Dump to file status
2549  * \retval -1 Error, and errno is set
2550  * \retval 0 Success
2551  */
2552 extern int mdbm_dbdump_to_file(kvpair kv, FILE *fp);
2553 
2554 /**
2555  * Export API: Finalize and close a file written in db_dump format and opened
2556  * with fopen. No trailer written: just closes the file (using fclose).
2557  *
2558  * \param[in,out] fp FILE pointer (return value of fopen)
2559  * \return Dump trailer status
2560  * \retval -1 Error, and errno is set
2561  * \retval 0 Success
2562  */
2563 extern int mdbm_dbdump_trailer_and_close(FILE *fp);
2564 
2565 /**
2566  * Export API: Use db_dump format to write a key/value pair to a malloc'd buffer.
2567  * mdbm_dbdump_add_record may realloc \a bufptr.
2568  *
2569  * \param[in] kv Key+Value pair
2570  * \param[in,out] datasize Size of db_dump data to be added added to \a bufptr
2571  * \param[in,out] bufptr malloc'd buffer. bufptr may be realloc'd if \a
2572  * datasize is greater than \a bufsize
2573  * \param[in] bufsize Size of malloc'd buffer
2574  * \param[in] buf_offset Offset in \a bufptr to start writing data
2575  * \return Dump add record status
2576  * \retval -1 Error, and errno is set
2577  * \retval 0 Success
2578  */
2579 extern int
2580 mdbm_dbdump_add_record(kvpair kv, uint32_t *datasize,
2581  char **bufptr, uint32_t *bufsize, uint32_t buf_offset);
2582 
2583 /**
2584  * Export API: Write the DBdump header to FILE
2585  *
2586  * \param[in,out] db handle of MDBM whose data is being exported
2587  * \param[in] fp file handle/pointer to read from
2588  */
2589 extern int mdbm_dbdump_export_header(MDBM *db, FILE *fp);
2590 
2591 /**
2592  * Import API: Read the DBdump header from FILE
2593  * Function will set, if available at the beginning of the file pointed to by "fp",
2594  * the following parameter values:
2595  *
2596  * \param[in] fp file handle/pointer to read from
2597  * \param[out] pgsize page-size read from "fp"
2598  * \param[out] pgcount page-count read from "fp"
2599  * \param[out] large large-object-support
2600  * \param[out] lineno The line number following the header (for tracking progress)
2601  */
2602 extern int
2603 mdbm_dbdump_import_header(FILE *fp, int *pgsize, int *pgcount, int *large, uint32_t *lineno);
2604 
2605 /**
2606  * Import API: Read data from FILE into MDBM, using DBdump format.
2607  * MDBM locking and unlocking is done one record at a time based on locking set up when
2608  * the MDBM was opened.
2609  *
2610  * \param[in,out] db handle of MDBM into which data is being imported
2611  * \param[in] fp file handle/pointer to read from
2612  * \param[in] input_file name of the file pointed to by "fp".
2613  * \param[in] store_flag MDBM_INSERT | MDBM_REPLACE | MDBM_INSERT_DUP | MDBM_MODIFY
2614  * \param[in,out] lineno current line number, keep track of the current line number
2615  * to generate meaningful error messages. Precondition: set to 1
2616  * or pass line number previously set by mdbm_dbdump_import_header.
2617  * Postcondition: lineno will hold the last line number read
2618  * successfully.
2619  */
2620 extern int
2621 mdbm_dbdump_import(MDBM *db, FILE *fp, const char *input_file, int store_flag, uint32_t *lineno);
2622 
2623 /**
2624  * Import API: Read data from FILE into MDBM, using Cdb format.
2625  * MDBM locking and unlocking is done one record at a time based on locking set up when
2626  * the MDBM was opened.
2627  *
2628  * \param[in,out] db handle of MDBM into which data is being imported
2629  * \param[in] fp file handle/pointer to read from
2630  * \param[in] input_file name of the file pointed to by "fp".
2631  * \param[in] store_flag MDBM_INSERT | MDBM_REPLACE | MDBM_INSERT_DUP | MDBM_MODIFY
2632  */
2633 extern int mdbm_cdbdump_import(MDBM *db, FILE *fp, const char *input_file, int store_flag);
2634 
2635 /** \} ImportExportGroup */
2636 
2637 /**
2638  * \defgroup MiscellaneousGroup Miscellaneous Group
2639  * The Miscellaneous Group contains the API for routines that don't clearly
2640  * fit into any of the other module API groups.
2641  * \{
2642  */
2643 
2644 /**
2645  * Returns the value of internally saved errno. Contains the value of errno
2646  * that is set during some lock failures. Under other circumstances,
2647  * mdbm_get_errno will not return the actual value of the errno variable.
2648  *
2649  * \param[in,out] db Database handle
2650  * \return Saved errno value, or zero if OK
2651  */
2652 extern int mdbm_get_errno(MDBM *db);
2653 
2654 /**
2655  * Given a hash function code, get the hash value for the given key.
2656  * See \ref mdbm_sethash for the list of valid hash function codes.
2657  *
2658  * \param[in] key Key
2659  * \param[in] hashFunctionCode for a valid hash function (below)
2660  * \param[out] hashValue is calculated according to the \a hashFunctionCode
2661  * \return Get Hash status
2662  * \retval -1 Error, and errno is set
2663  * \retval 0 Success
2664  *
2665  * Values for \a hashFunctionCode:
2666  * - MDBM_HASH_CRC32 - Table based 32bit CRC
2667  * - MDBM_HASH_EJB - From hsearch
2668  * - MDBM_HASH_PHONG - Congruential hash
2669  * - MDBM_HASH_OZ - From sdbm
2670  * - MDBM_HASH_TOREK - From BerkeleyDB
2671  * - MDBM_HASH_FNV - Fowler/Vo/Noll hash (DEFAULT)
2672  * - MDBM_HASH_STL - STL string hash
2673  * - MDBM_HASH_MD5 - MD5
2674  * - MDBM_HASH_SHA_1 - SHA_1
2675  * - MDBM_HASH_JENKINS - Jenkins string
2676  * - MDBM_HASH_HSIEH - Hsieh SuperFast
2677  */
2678 extern int mdbm_get_hash_value(datum key, int hashFunctionCode, uint32_t *hashValue);
2679 
2680 /**
2681  * Gets the MDBM page number for a given key.
2682  * The key does not actually have to exist.
2683  *
2684  * \param[in,out] db Database handle
2685  * \param[in] key Lookup key
2686  * \return Page number or error indicator
2687  * \retval -1 Error, and errno is set
2688  * \retval Page number where the parameter key would be stored.
2689  */
2690 extern mdbm_ubig_t mdbm_get_page(MDBM *db, const datum *key);
2691 
2692 /**
2693  * Preload mdbm: Read every 4k bytes to force all pages into memory
2694  *
2695  * \param[in,out] db Database handle
2696  * \return preload status
2697  * \retval -1 Error
2698  * \retval 0 Success
2699  */
2700 extern int mdbm_preload(MDBM* db);
2701 
2702 /**
2703  * mdbm_lock_pages: Locks MDBM data pages into memory.
2704  *
2705  * When running MDBM as root, mdbm_lock_pages will expand the amount of RAM that can be locked to
2706  * infinity using setrlimit(RLIMIT_MEMLOCK). When not running as root, mdbm_lock_pages will expand
2707  * the amount of RAM that can be locked up to the maximum allowed (retrieved using getrlimit(MEMLOCK),
2708  * and normally a very small amount), and if the MDBM is larger than the amount of RAM that can be
2709  * locked, a warning will be logged but mdbm_lock_pages will return 0 for success.
2710  *
2711  * \param[in,out] db Database handle
2712  * \return lock pages status
2713  * \retval -1 Error
2714  * \retval 0 Success
2715  */
2716 extern int mdbm_lock_pages(MDBM* db);
2717 
2718 /**
2719  * mdbm_unlock_pages: Releases MDBM data pages from always staying in memory
2720  *
2721  * \param[in,out] db Database handle
2722  * \return unlock pages status
2723  * \retval -1 Error
2724  * \retval 0 Success
2725  */
2726 extern int mdbm_unlock_pages(MDBM* db);
2727 
2728 /** \} MiscellaneousGroup */
2729 
2730 
2731 
2732 /*
2733  * TODO
2734  * TODO Merge remnants are here. Verify where they should go.
2735  * TODO
2736  */
2737 
2738 
2739 
2740 #define MDBM_MINPAGE 128 /**< Size should be >= header, but this cuts off header stats, which is ok */
2741 #define MDBM_PAGE_ALIGN 64
2742 
2743 /* Maximum page size is 16MB-64bytes, because any page size above that would be rounded to 16MB,
2744  * which does not fit in the 24 bits allocated for the in-page offset to an object */
2745 #define MDBM_MAXPAGE ((16*1024*1024) - MDBM_PAGE_ALIGN)
2746 
2747 #define MDBM_PAGESIZ 4096 /**< A good default. Size should be >= header */
2748 #define MDBM_MIN_PSHIFT 7 /**< this must match MDBM_MINPAGE above */
2749 #define MDBM_MAX_SHIFT ((sizeof( void * )*8)-1) /**< Base # shifts on ptr size */
2750 
2751 extern void mdbm_lock_dump(MDBM* db);
2752 
2753 
2754 int mdbm_get_db_stats(MDBM* db, mdbm_db_info_t* dbinfo, mdbm_stat_info_t* dbstats, int flags);
2755 
2756 
2757 
2758 #ifdef LINT
2759 # define WHATSTR(X) pid_t getpid(void)
2760 #else
2761 # define WHATSTR(X) static const char what[] = X
2762 #endif
2763 
2764 
2765 /* If we're using OpenSSL, map the FreeBSD symbol names (used
2766 * within mdbm) to match the OpenSSL symbol names. */
2767 #ifdef USE_OPENSSL
2768 # define MD5Init MD5_Init
2769 # define MD5Final MD5_Final
2770 # define MD5Update MD5_Update
2771 #endif /* USE_OPENSSL */
2772 
2773 
2774 
2775 typedef mdbm_ubig_t ((* mdbm_hash_t)(const uint8_t *buf, int len));
2776 extern mdbm_hash_t mdbm_hash_funcs[];
2777 
2778 /*
2779  * Hash functions
2780  */
2781 #define MDBM_HASH_CRC32 0 /**< table based 32bit crc */
2782 #define MDBM_HASH_EJB 1 /**< from hsearch */
2783 #define MDBM_HASH_PHONG 2 /**< congruential hash */
2784 #define MDBM_HASH_OZ 3 /**< from sdbm */
2785 #define MDBM_HASH_TOREK 4 /**< from Berkeley db */
2786 #define MDBM_HASH_FNV 5 /**< Fowler/Vo/Noll hash */
2787 #define MDBM_HASH_STL 6 /**< STL string hash */
2788 #define MDBM_HASH_MD5 7 /**< MD5 */
2789 #define MDBM_HASH_SHA_1 8 /**< SHA_1 */
2790 #define MDBM_HASH_JENKINS 9 /**< JENKINS */
2791 #define MDBM_HASH_HSIEH 10 /**< HSIEH SuperFastHash */
2792 #define MDBM_MAX_HASH 10 /* bump up if adding more */
2793 
2794 /** Define the hash function to use on a newly created file */
2795 #ifndef MDBM_DEFAULT_HASH
2796 # define MDBM_DEFAULT_HASH 5 /* MDBM_HASH_FNV is best */
2797 #endif
2798 
2799 #define MDBM_CONFIG_DEFAULT_HASH MDBM_HASH_FNV
2800 
2801 
2802 #define MDBM_BAD_HASH_NO(n) ((n) < 0 || (n) > MDBM_MAX_HASH)
2803 
2804 typedef uint32_t mdbm_hashval_t;
2805 /*typedef mdbm_hashval_t (*mdbm_hash_t)(const unsigned char *buf, int len); */
2806 /*extern mdbm_hash_t mdbm_hash_funcs[]; */
2807 /*#define MDBM_HASH_MD5 7 */ /* MD5 */
2808 /*#define MDBM_HASH_SHA_1 8 */ /* SHA_1 */
2809 /*#define MDBM_MAX_HASH 10 */
2810 #define MDBM_HASH_MAX MDBM_MAX_HASH
2811 
2812 /* Which hash function to use on a newly created file */
2813 /*#define MDBM_DEFAULT_HASH MDBM_CONFIG_DEFAULT_HASH */
2814 
2815 /* count_all_page() commented out: use mdbm_count_records() instead */
2816 /* extern mdbm_ubig_t count_all_page(MDBM *db); */
2817 
2818 /* optional - may be user supplied. */
2819 extern mdbm_ubig_t mdbm_hash(unsigned char *, int);
2820 extern mdbm_ubig_t mdbm_hash0(unsigned char *, int);
2821 extern mdbm_ubig_t mdbm_hash1(unsigned char *, int);
2822 extern mdbm_ubig_t mdbm_hash2(unsigned char *, int);
2823 extern mdbm_ubig_t mdbm_hash3(unsigned char *, int);
2824 extern mdbm_ubig_t mdbm_hash4(unsigned char *, int);
2825 extern mdbm_ubig_t mdbm_hash5(unsigned char *, int);
2826 extern mdbm_ubig_t mdbm_hash6(unsigned char *, int);
2827 
2828 
2829 #ifdef __cplusplus
2830 }
2831 #endif
2832 
2833 #endif /* __MDBM_H__ */
2834