diff --git a/3P_COPYRIGHT.md b/3P_COPYRIGHT.md new file mode 100644 index 0000000..346892c --- /dev/null +++ b/3P_COPYRIGHT.md @@ -0,0 +1,22 @@ +Portions of this code are based on the ULID implementation at +https://github.com/dylanhart/ulid-rs/tree/0b9295c2db2114cd87aa19abcc1fc00c16b272db and used under +the terms of the MIT license: + +Copyright (c) 2017 Dylan Hart + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES +OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Cargo.toml b/Cargo.toml index d606eee..34c26c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,19 @@ [package] -name = "julid" -version = "0.1.0" +name = "julid-rs" +version = "0.0.1" +authors = ["Joe Ardent "] edition = "2021" +keywords = ["ulid", "library", "sqlite", "extension"] + +description = "A library and loadable extension for SQLite that uses it, that provides Joe's ULIDs." +readme = "README.md" +license-file = "LICENSE.md" +repository = "https://gitlab.com/nebkor/julid" + +[lib] +name = "julid" +crate-type = ["cdylib", "rlib"] [dependencies] -bitfield = "0.14.0" -rand = "0.8.5" +rand = "0.8" +sqlite-loadable = "0.0.5" diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..589cfb6 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,5 @@ +# The Chaos License (GLP) + +This software is released under the terms of the Chaos License. In cases where the terms of the +license are unclear, refer to the [Fuck Around and Find Out +License](https://git.sr.ht/~boringcactus/fafol/tree/master/LICENSE-v0.2.md). diff --git a/README.md b/README.md new file mode 100644 index 0000000..c856b0c --- /dev/null +++ b/README.md @@ -0,0 +1,101 @@ +Globally unique sortable identifiers for SQLite! + +# quick take + +``` text +$ sqlite3 +SQLite version 3.40.1 2022-12-28 14:03:47 +Enter ".help" for usage hints. +Connected to a transient in-memory database. +Use ".open FILENAME" to reopen on a persistent database. +sqlite> .load ./libjulid +sqlite> select hex(julid_new()); +01898F1332B90000D6F0F0FE1066A6BF +sqlite> select julid_string(julid_new()); +01H67GV14N000BBHJD6FARVB7Q +sqlite> select datetime(julid_timestamp(julid_new()) / 1000, 'auto'); -- sqlite wants seconds, not milliseconds +2023-07-25 21:58:56 +sqlite> select julid_counter(julid_new()); +0 +``` + +## a little more in depth + +Julids are [ULID](https://github.com/ulid/spec)-backwards-compatible (that is, all Julids are valid ULIDs, +but not all ULIDs are Julids) identifiers with the following properties: + + * they are 128-bits long + * they are lexicographically sortable + * they encode their creation time as the number of milliseconds since the UNIX epoch + * IDs created within the same millisecond will still sort in their order of creation, due to the + presence of a 16-bit monotonic counter, placed immediately after the creation time bits + +It's that last bit that makes them distinctive. ULIDs have the following big-endian bit structure: + +![ULID bit structure][./ulid.svg] + +According to the ULID spec, for ULIDs created in the same millisecond, the least-significant bit +should be incremented for each new one. Since that portion of the ULID is random, that means you may +not be able to increment it without spilling into the timestamp portion. Likewise, it's easy to +guess a new possibly-valid ULID simply by incrementing an already-known one. And finally, this means +that sorting will need to read all the way to the end of the ULID for IDs created in the same +millisecond. + +To address these shortcomings, Julids (Joe's ULIDs) have the following big-endian bit structure: + +![Julid bit structure][./julid.svg] + +As with ULIDs, the 48 most-significant bits encode the time of creation. Unlike ULIDs, the next 16 +bits are not random, they're a monotonic counter for IDs created within the same millisecond. Since +it's only 16 bits, it will saturate after 65,536 IDs intra-millisecond creations, after which, IDs +in that same millisecond will not have an intrinsic total order (the random bits will still be +different, so you shouldn't have collisions). My PC, which is no slouch, can only generate about +20,000 per millisecond, so hopefully this is not an issue! Because the random bits are always fresh, +it is not possible to guess a valid Julid if you already know one. + +# functions overview + +This extension provides the following functions: + + * `julid_new()`: create a new Julid and return it as a `blob` + * `julid_timestamp(julid)`: get the number milliseconds since the UNIX epoch that this julid was + created + * `julid_counter(julid)`: show the value of this julid's monotonic counter + * `julid_sortable(julid)`: return the 64-bit concatenation of the timestamp and counter + * `julid_string(julid)`: show the [base-32 Crockford](https://en.wikipedia.org/wiki/Base32) + encoding of this julid + +# how to use + + * clone the repo + * build it with `cargo build` + * copy the resulting `libjulid.[so|dylib|whatevs]` to some place where you can... + * load it into SQLite with `.load /path/to/libjulid` as shown at the top + * party + +If you, like me, wish to use Julids as primary keys, just create your table like: + +``` sql +create table users ( + id blob not null primary key default julid_new(), + ... +); +``` + +and you've got a first-class ticket straight to Julid City, baby! + +## using it as a library in a Rust application + +Of course, you can also use it outside of a database; the `Julid` type is publicly exported, and +you can do like such as: + +``` rust +use julid::Julid; + +fn main() { + let id = Julid::new(); + dbg!(id.timestamp(), id.counter(), id.sortable(), id.as_string()); +} +``` + +after adding it to your project's dependencies, like `cargo add julid-rs`. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1 diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 0000000..5dfd982 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,32 @@ +# Golden Versioning + +This software is versioned under a scheme I call "goldver", as an homage to the +vastly inferior [semver](https://semver.org). + +## What does "goldver" mean? + +When projects are versioned with goldver, the first version is "1". Note that it +is not "1.0", or, "1.0-prealpha-release-preview", or anything nonsensical like +that. As new versions are released, decimals from *phi*, the [Golden +Ratio](https://en.wikipedia.org/wiki/Golden_ratio), are appended after an +initial decimal point. So the second released version will be "1.6", the third +would be "1.61", etc., and on until perfection is asymptotically approached as +the number of released versions goes to infinity. + +## Wait, didn't Donald Knuth do this? + +No! He uses [pi for TeX and e for MetaFont](https://texfaq.org/FAQ-TeXfuture), +obviously COMPLETELY different. + +## Ok. + +Cool. + +## What version is Julid now? + +Canonically, see the `VERSION` file. Heretically, once there have been +at least three releases, the version string in the `Cargo.toml` file will +always be of the form "1.6.x", where *x* is at least one digit long, starting +with "1". Each subsequent release will append the next digit of *phi* to +*x*. The number of releases can be calculated by counting the number of digits +in *x* and adding 2 to that. diff --git a/julid.svg b/julid.svg new file mode 100644 index 0000000..2be292d --- /dev/null +++ b/julid.svg @@ -0,0 +1 @@ +0474863timestampmonotonic counter64127random diff --git a/src/base32.rs b/src/base32.rs index e9a8c38..a875efb 100644 --- a/src/base32.rs +++ b/src/base32.rs @@ -1,35 +1,7 @@ -/* -this code shamelessly mostly stolen from -https://github.com/dylanhart/ulid-rs/blob/0b9295c2db2114cd87aa19abcc1fc00c16b272db/src/base32.rs -and used under the terms of the MIT license: - -Copyright (c) 2017 Dylan Hart - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - use core::fmt; -/// Length of a string-encoded Ulid -pub const ULID_LEN: usize = 26; +/// Length of a string-encoded Julid +pub const JULID_LEN: usize = 26; const ALPHABET: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ"; @@ -64,11 +36,11 @@ lookup[(c + 32) as usize] = i as u8; } */ -/// Encode the given 128-bit number as a base32 string. +/// Encode the given 128-bit little-endian number as a base32 string. pub fn encode(mut value: u128) -> String { - let mut buffer: [u8; ULID_LEN] = [0; ULID_LEN]; - for i in 0..ULID_LEN { - buffer[ULID_LEN - 1 - i] = ALPHABET[(value & 0x1f) as usize]; + let mut buffer: [u8; JULID_LEN] = [0; JULID_LEN]; + for i in 0..JULID_LEN { + buffer[JULID_LEN - 1 - i] = ALPHABET[(value & 0x1f) as usize]; value >>= 5; } @@ -98,7 +70,7 @@ impl fmt::Display for DecodeError { pub const fn decode(encoded: &str) -> Result { let len = encoded.len(); - if len != ULID_LEN { + if len != JULID_LEN { return Err(DecodeError::InvalidLength(len)); } @@ -108,7 +80,7 @@ pub const fn decode(encoded: &str) -> Result { // Manual for loop because Range::iter() isn't const let mut i = 0; - while i < ULID_LEN { + while i < JULID_LEN { let val = LOOKUP[bytes[i] as usize]; if val != NO_VALUE { value = (value << 5) | val as u128; @@ -141,9 +113,9 @@ mod tests { #[test] fn test_length() { - assert_eq!(encode(0xffffffffffffffffffffffffffffffff).len(), ULID_LEN); - assert_eq!(encode(0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f).len(), ULID_LEN); - assert_eq!(encode(0x00000000000000000000000000000000).len(), ULID_LEN); + assert_eq!(encode(0xffffffffffffffffffffffffffffffff).len(), JULID_LEN); + assert_eq!(encode(0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f).len(), JULID_LEN); + assert_eq!(encode(0x00000000000000000000000000000000).len(), JULID_LEN); assert_eq!(decode(""), Err(DecodeError::InvalidLength(0))); assert_eq!( diff --git a/src/julid.rs b/src/julid.rs index f87f883..3f3edae 100644 --- a/src/julid.rs +++ b/src/julid.rs @@ -1,7 +1,7 @@ use core::{fmt, str::FromStr}; use std::{sync::Mutex, time::Duration}; -use rand::random; +use rand::{random, thread_rng, Rng}; use crate::base32::{self, DecodeError}; @@ -14,7 +14,7 @@ pub const TIME_BITS: u8 = 48; pub const MBITS: u8 = 16; /// The number of random bits + bits in the monotonic counter pub const UNIQUE_BITS: u8 = 80; -pub const RANDOM_BITS: u8 = UNIQUE_BITS - MBITS; +pub const RANDOM_BITS: u8 = 64; macro_rules! bitmask { ($len:expr) => { @@ -44,8 +44,7 @@ impl Julid { pub fn new() -> Self { let lsb: u64 = random(); loop { - let guard = LAST_ID.try_lock(); - if let Ok(mut guard) = guard { + if let Ok(mut guard) = LAST_ID.try_lock() { let ts = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or(Duration::ZERO) @@ -64,7 +63,8 @@ impl Julid { break new; } } - std::thread::sleep(Duration::from_micros(50)); + let micros = thread_rng().gen_range(10..50); + std::thread::sleep(Duration::from_micros(micros)); } } @@ -264,10 +264,11 @@ mod tests { #[test] fn can_increment() { let mut max = 0; - for _ in 0..100 { + for i in 0..100 { let id = Julid::new(); max = id.counter().max(max); + assert!(max <= i); } - assert!(max > 0); + assert!(max > 49); } } diff --git a/src/lib.rs b/src/lib.rs index e00d4d7..ebe4c18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,116 @@ +use sqlite_loadable::{ + api, define_scalar_function, + prelude::{ + c_char, c_uint, sqlite3, sqlite3_api_routines, sqlite3_context, sqlite3_value, + FunctionFlags, + }, + Result, +}; + mod base32; pub mod julid; pub use julid::Julid; + +//-************************************************************************ +// Entrypoint into the loadable extension +//-************************************************************************ +#[no_mangle] +pub unsafe extern "C" fn sqlite3_julid_init( + db: *mut sqlite3, + _pz_err_msg: *mut *mut c_char, + p_api: *mut sqlite3_api_routines, +) -> c_uint { + unsafe { sqlite_loadable::ext::faux_sqlite_extension_init2(p_api) } + match init_rs(db) { + Ok(()) => 256, // SQLITE_OK_LOAD_PERMANENTLY + Err(err) => err.code_extended(), + } +} + +fn init_rs(db: *mut sqlite3) -> Result<()> { + let flags = FunctionFlags::UTF8 | FunctionFlags::DETERMINISTIC; + define_scalar_function(db, "julid_new", 0, julid_new, FunctionFlags::INNOCUOUS)?; + define_scalar_function(db, "julid_timestamp", 1, julid_timestamp, flags)?; + define_scalar_function(db, "julid_counter", 1, julid_counter, flags)?; + define_scalar_function(db, "julid_sortable", 1, julid_sortable, flags)?; + define_scalar_function(db, "julid_string", 1, julid_string, flags)?; + + Ok(()) +} + +//-************************************************************************ +// impls +//-************************************************************************ +fn julid_new(context: *mut sqlite3_context, _vals: &[*mut sqlite3_value]) -> Result<()> { + api::result_blob(context, Julid::new().to_bytes().as_slice()); + Ok(()) +} + +fn julid_timestamp(context: *mut sqlite3_context, id: &[*mut sqlite3_value]) -> Result<()> { + if let Some(value) = id.get(0) { + let id = api::value_blob(value); + let bytes: [u8; 16] = id.try_into().map_err(|_| { + sqlite_loadable::Error::new_message("Could not convert given value to Julid") + })?; + let id: Julid = bytes.into(); + api::result_int64(context, id.timestamp() as i64); + } else { + return Err(sqlite_loadable::Error::new_message( + "Could not get timestamp for empty Julid", + )); + } + + Ok(()) +} + +fn julid_counter(context: *mut sqlite3_context, id: &[*mut sqlite3_value]) -> Result<()> { + if let Some(value) = id.get(0) { + let id = api::value_blob(value); + let bytes: [u8; 16] = id.try_into().map_err(|_| { + sqlite_loadable::Error::new_message("Could not convert given value to Julid") + })?; + let id: Julid = bytes.into(); + api::result_int64(context, id.counter() as i64); + } else { + return Err(sqlite_loadable::Error::new_message( + "Could not get counter value for empty Julid", + )); + } + + Ok(()) +} + +fn julid_sortable(context: *mut sqlite3_context, id: &[*mut sqlite3_value]) -> Result<()> { + if let Some(value) = id.get(0) { + let id = api::value_blob(value); + let bytes: [u8; 16] = id.try_into().map_err(|_| { + sqlite_loadable::Error::new_message("Could not convert given value to Julid") + })?; + let id: Julid = bytes.into(); + api::result_int64(context, id.sortable() as i64); + } else { + return Err(sqlite_loadable::Error::new_message( + "Could not get sortable bits for empty Julid", + )); + } + + Ok(()) +} + +fn julid_string(context: *mut sqlite3_context, id: &[*mut sqlite3_value]) -> Result<()> { + if let Some(value) = id.get(0) { + let id = api::value_blob(value); + let bytes: [u8; 16] = id.try_into().map_err(|_| { + sqlite_loadable::Error::new_message("Could not convert given value to Julid") + })?; + let id: Julid = bytes.into(); + api::result_text(context, id.as_string())?; + } else { + return Err(sqlite_loadable::Error::new_message( + "Could not convert empty Julid to string", + )); + } + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 71a7930..00f00f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use julid::julid::Julid; +use julid::Julid; fn main() { let mut v = Vec::with_capacity(2000); diff --git a/ulid.svg b/ulid.svg new file mode 100644 index 0000000..96d29b0 --- /dev/null +++ b/ulid.svg @@ -0,0 +1 @@ +0474863timestamprandom64127random