Add models, use kv store for storage of projects/documents, and add proc

macro to derive Model automatically. Projects can be created!
This commit is contained in:
Nicole Tietz-Sokolskaya 2024-05-09 12:15:19 -04:00
parent ff76f90f88
commit 9e98f86fe3
17 changed files with 809 additions and 173 deletions

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
target/ target/
kvdata/
node_modules/ node_modules/
*.db *.db
*.xml *.xml

502
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -17,13 +17,16 @@ bincode = "1.3.3"
clap = { version = "4.5.3", features = ["derive", "env"] } clap = { version = "4.5.3", features = ["derive", "env"] }
dotenvy = "0.15.7" dotenvy = "0.15.7"
env_logger = "0.11.3" env_logger = "0.11.3"
fjall = "0.6.5"
free-icons = "0.7.0" free-icons = "0.7.0"
minijinja = { version = "1.0.14", features = ["loader"] } minijinja = { version = "1.0.14", features = ["loader"] }
minijinja-autoreload = "1.0.14" minijinja-autoreload = "1.0.14"
model_derive = { path = "./model_derive" }
rand = "0.8.5" rand = "0.8.5"
redb = "2.1.0"
sea-orm = { version = "0.12.15", features = ["sqlx-sqlite", "macros", "runtime-tokio-rustls"] } sea-orm = { version = "0.12.15", features = ["sqlx-sqlite", "macros", "runtime-tokio-rustls"] }
serde = { version = "1.0.197", features = ["derive"] } serde = { version = "1.0.197", features = ["derive"] }
sled = "0.34.7" sled = "=1.0.0-alpha.121"
thiserror = "1.0.58" thiserror = "1.0.58"
tokio = { version = "1.36.0", features = ["rt", "full"] } tokio = { version = "1.36.0", features = ["rt", "full"] }
tower-http = { version = "0.5.2", features = ["fs", "trace"] } tower-http = { version = "0.5.2", features = ["fs", "trace"] }
@ -32,3 +35,4 @@ tower-sessions-moka-store = "0.11.0"
tower-sessions-sqlx-store = { version = "0.11.0", features = ["sqlite"] } tower-sessions-sqlx-store = { version = "0.11.0", features = ["sqlite"] }
tracing = "0.1.40" tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
uuid = { version = "1.8.0", features = ["v4", "fast-rng", "v7", "serde"] }

View file

@ -2,6 +2,9 @@
run: run:
SECURE_SESSIONS=false RUST_LOG=debug cargo run -- --reload-templates SECURE_SESSIONS=false RUST_LOG=debug cargo run -- --reload-templates
run-watch:
SECURE_SESSIONS=false RUST_LOG=debug cargo watch --why -x 'run -- --reload-templates'
run-release: run-release:
SECURE_SESSIONS=false RUST_LOG=info cargo run --release SECURE_SESSIONS=false RUST_LOG=info cargo run --release

46
model_derive/Cargo.lock generated Normal file
View file

@ -0,0 +1,46 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "model_derive"
version = "0.1.0"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "syn"
version = "2.0.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

11
model_derive/Cargo.toml Normal file
View file

@ -0,0 +1,11 @@
[package]
name = "model_derive"
version = "0.1.0"
edition = "2021"
[lib]
proc-macro = true
[dependencies]
quote = "1.0.36"
syn = { version = "2.0.61", features = ["full", "derive"] }

48
model_derive/src/lib.rs Normal file
View file

@ -0,0 +1,48 @@
use proc_macro::TokenStream;
use quote::{format_ident, quote};
use syn::{parse_macro_input, DeriveInput, LitInt};
#[proc_macro_derive(Model, attributes(model_version))]
pub fn model(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as DeriveInput);
let name = input.ident;
let attrs = input.attrs;
let model_version = attrs.iter().find_map(|attr| {
if attr.path().is_ident("model_version") {
attr.parse_args::<LitInt>().ok().and_then(|lit| {
Some(lit.base10_parse::<u64>().unwrap())
})
} else {
None
}
}).unwrap_or(0);
let lower_name = name.to_string().to_lowercase();
let lower_name_ident = format_ident!("{}", lower_name);
let expanded = quote! {
impl Model for #name {
type Id = uuid::Uuid;
fn id(&self) -> Self::Id {
self.id
}
fn key(id: Self::Id) -> Vec<u8> {
let mut key = vec![];
key.extend_from_slice(format!("{}:{}:", #lower_name, #model_version).as_bytes());
key.extend_from_slice(&id.into_bytes());
key
}
fn partition(kv_handle: &KvHandle) -> &PartitionHandle {
&kv_handle.#lower_name_ident
}
}
};
TokenStream::from(expanded)
}

View file

@ -2,18 +2,21 @@ use std::sync::Arc;
use minijinja_autoreload::AutoReloader; use minijinja_autoreload::AutoReloader;
use crate::{handler::internal_server_error, prelude::*}; use crate::{handler::internal_server_error, kv::KvHandle, prelude::*};
#[derive(Clone)] #[derive(Clone)]
pub struct Context { pub struct Context {
pub db: DatabaseConnection, pub db: DatabaseConnection,
// TODO: add a design doc explaining why this not relational
pub kv_handles: KvHandle,
template_loader: Arc<AutoReloader>, template_loader: Arc<AutoReloader>,
} }
impl Context { impl Context {
pub fn new(db: DatabaseConnection, template_loader: AutoReloader) -> Context { pub fn new(db: DatabaseConnection, kv_handles: KvHandle, template_loader: AutoReloader) -> Context {
Context { Context {
db, db,
kv_handles,
template_loader: Arc::new(template_loader), template_loader: Arc::new(template_loader),
} }
} }

View file

@ -1,26 +1,11 @@
use axum::response::Redirect; use axum::response::Redirect;
use axum_login::AuthSession; use axum_login::AuthSession;
use crate::{models::Project, prelude::*}; use crate::{models::{ModelPermission, Project}, prelude::*};
pub async fn home_page(State(ctx): State<Context>, auth_session: AuthSession<Context>) -> Response { pub async fn home_page(State(ctx): State<Context>, auth_session: AuthSession<Context>) -> Response {
if let Some(user) = auth_session.user { if let Some(user) = auth_session.user {
let projects: Vec<Project> = vec![ let projects: Vec<Project> = ModelPermission::user_projects(&ctx.kv_handles, user.id).unwrap_or_default();
Project {
id: 1,
owner_id: 1,
name: "Blog posts".to_owned(),
description: "Planning and publication schedule for my blog".to_owned(),
key: "BLOG".to_owned(),
},
Project {
id: 2,
owner_id: 1,
name: "Bugs (Pique)".to_owned(),
description: "The bugs we've found so far in Pique".to_owned(),
key: "BUG".to_owned(),
},
];
let values = context! { let values = context! {
user => user, user => user,

View file

@ -1,50 +1,92 @@
use axum::response::Redirect; use axum::{response::Redirect, Form};
use axum_login::AuthSession; use axum_login::AuthSession;
use crate::{models::Project, prelude::*}; use crate::{
handler::internal_server_error,
models::{ModelPermission, ModelType, Permission, Project},
prelude::*,
};
pub async fn projects_page( pub async fn projects_page(
State(ctx): State<Context>, State(ctx): State<Context>,
auth_session: AuthSession<Context>, auth_session: AuthSession<Context>,
) -> Response { ) -> Response {
if let Some(user) = auth_session.user { if let Some(user) = auth_session.user {
let projects: Vec<Project> = vec![ render_projects_page(ctx, user).await
Project {
id: 1,
owner_id: 1,
name: "Blog posts".to_owned(),
description: "Planning and publication schedule for my blog".to_owned(),
key: "BLOG".to_owned(),
},
Project {
id: 2,
owner_id: 1,
name: "Bugs (Pique)".to_owned(),
description: "The bugs we've found so far in Pique".to_owned(),
key: "BUG".to_owned(),
},
];
let values = context! {
user => user,
projects => projects,
};
ctx.render_resp("projects/list_projects.html", values)
} else { } else {
Redirect::to("/login").into_response() Redirect::to("/login").into_response()
} }
} }
pub async fn create_project( async fn render_projects_page(ctx: Context, user: crate::entity::user::Model) -> Response {
let projects = ModelPermission::user_projects(&ctx.kv_handles, user.id).unwrap_or_default();
let values = context! {
user => user,
projects => projects,
};
ctx.render_resp("projects/list_projects.html", values)
}
pub async fn create_project_page(
State(ctx): State<Context>, State(ctx): State<Context>,
auth_session: AuthSession<Context>, auth_session: AuthSession<Context>,
) -> Response { ) -> Response {
if let Some(_user) = auth_session.user { let user = match auth_session.user {
let values = context! {}; Some(user) => user,
None => return Redirect::to("/login").into_response(),
};
ctx.render_resp("projects/create_project.html", values) let values = context! {
} else { user => user,
Redirect::to("/login").into_response() };
} ctx.render_resp("projects/create_project.html", values)
}
#[derive(Debug, Deserialize)]
pub struct CreateProjectSubmission {
pub name: String,
pub key: String,
pub description: String,
}
pub async fn create_project_submit(
State(ctx): State<Context>,
auth_session: AuthSession<Context>,
form: Form<CreateProjectSubmission>,
) -> Response {
let user = match auth_session.user {
Some(user) => user,
None => return Redirect::to("/login").into_response(),
};
let project = Project {
id: Uuid::new_v4(),
owner_id: user.id,
name: form.name.clone(),
key: form.key.clone(),
description: form.description.clone(),
};
// TODO: validation
if let Err(err) = project.save(&ctx.kv_handles) {
error!(?err, "failed to save new project");
return internal_server_error();
}
let permission = ModelPermission {
user_id: user.id,
model_type: ModelType::Project,
role: Permission::Admin,
model_id: project.id,
};
if let Err(err) = permission.add(&ctx.kv_handles) {
error!(?err, "failed to save new project permission");
return internal_server_error();
}
Redirect::to("/projects").into_response()
} }

42
src/kv.rs Normal file
View file

@ -0,0 +1,42 @@
use std::path::Path;
use anyhow::Result;
use fjall::{Config, Keyspace, PartitionCreateOptions, PartitionHandle};
/// Contains the handles needed to reference key-value data.
///
/// This contains both the Keyspace and multiple PartitionHandle.
/// The Keyspace allows operational control and reporting at the top level,
/// while each PartitionHandle controls reading, writing, and removing from a
/// particular partition of the data.
///
/// All fields are public, because this is meant to be used internally as a
/// wrapper to pass everything around, instead of passing each handle around by
/// itself.
#[derive(Clone)]
pub struct KvHandle {
pub keyspace: Keyspace,
pub project: PartitionHandle,
pub document: PartitionHandle,
pub permissions: PartitionHandle,
}
impl KvHandle {
pub fn open<P: AsRef<Path>>(p: P) -> Result<KvHandle> {
// TODO: those should probably be configurable, or like, not just hard coded.
let config = Config::new(p).flush_workers(4).compaction_workers(4);
let keyspace = Keyspace::open(config)?;
let project = keyspace.open_partition("project", PartitionCreateOptions::default())?;
let document = keyspace.open_partition("document", PartitionCreateOptions::default())?;
let permissions = keyspace.open_partition("permissions", PartitionCreateOptions::default())?;
Ok(KvHandle {
keyspace,
project,
document,
permissions,
})
}
}

View file

@ -11,3 +11,4 @@ pub mod serialize;
pub mod server; pub mod server;
pub mod session; pub mod session;
pub mod templates; pub mod templates;
pub mod kv;

View file

@ -1,5 +1,7 @@
use tracing_subscriber::EnvFilter;
pub fn setup_logging() { pub fn setup_logging() {
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG) .with_env_filter(EnvFilter::from_default_env())
.init(); .init();
} }

View file

@ -1,8 +1,50 @@
use serde::{Deserialize, Serialize}; use core::fmt::{self, Display};
use crate::prelude::*;
#[derive(Debug, Serialize, Deserialize, PartialEq)] use anyhow::Result;
use fjall::PartitionHandle;
use model_derive::Model;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::kv::KvHandle;
pub trait Model: Sized + Serialize + for<'a> Deserialize<'a> {
type Id;
fn id(&self) -> Self::Id;
fn key(id: Self::Id) -> Vec<u8>;
fn partition(kv_handle: &KvHandle) -> &PartitionHandle;
fn save(&self, kv_handle: &KvHandle) -> Result<()> {
let key = Self::key(self.id());
let value = bincode::serialize(self)?;
let partition = Self::partition(kv_handle);
partition.insert(key, value)?;
Ok(())
}
fn load(kv_handle: &KvHandle, id: Self::Id) -> Result<Option<Self>> {
let key = Self::key(id);
let partition = Self::partition(kv_handle);
match partition.get(key.as_slice())? {
Some(bytes) => {
let bytes = bytes.to_vec();
let value: Self = bincode::deserialize(&bytes)?;
Ok(Some(value))
}
None => Ok(None),
}
}
}
#[derive(Debug, Model, Serialize, Deserialize, PartialEq)]
#[model_version(0)]
pub struct Project { pub struct Project {
pub id: u64, pub id: Uuid,
pub owner_id: i32, pub owner_id: i32,
pub name: String, pub name: String,
@ -14,11 +56,106 @@ pub struct Project {
pub key: String, pub key: String,
} }
#[derive(Debug, Serialize, Deserialize, PartialEq)] #[derive(Debug, Model, Serialize, Deserialize, PartialEq)]
#[model_version(0)]
pub struct Document { pub struct Document {
pub id: u64, pub id: Uuid,
pub project_id: u64, pub project_id: Uuid,
pub title: String, pub title: String,
pub content: String, pub content: String,
} }
#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[repr(u8)]
pub enum ModelType {
Project = 0,
Document = 1,
}
impl Display for ModelType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ModelType::Project => write!(f, "project"),
ModelType::Document => write!(f, "document"),
}
}
}
#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[repr(u8)]
pub enum Permission {
Admin = 0,
Read = 1,
}
impl Display for Permission {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Permission::Admin => write!(f, "admin"),
Permission::Read => write!(f, "read"),
}
}
}
#[derive(Debug, Serialize, Deserialize, PartialEq)]
pub struct ModelPermission {
pub user_id: i32,
pub model_type: ModelType,
pub role: Permission,
pub model_id: Uuid,
}
impl ModelPermission {
pub fn add(&self, kv_handle: &KvHandle) -> Result<()> {
let key = format!(
"{}:{}:{}:{}",
self.user_id, self.model_type, self.role, self.model_id
);
let value = bincode::serialize(self)?;
kv_handle.permissions.insert(key, value)?;
Ok(())
}
pub fn user_projects(kv_handle: &KvHandle, user_id: i32) -> Result<Vec<Project>> {
let prefix = format!("{}:{}:", user_id, ModelType::Project);
let mut ids = vec![];
for row in kv_handle.permissions.prefix(prefix).into_iter() {
let (_key, value) = row?;
let permission: ModelPermission = bincode::deserialize(&value)?;
ids.push(permission.model_id);
}
let projects: Vec<Project> = ids
.into_iter()
.filter_map(|id| {
let res = Project::load(kv_handle, id);
res.ok().flatten()
})
.collect();
Ok(projects)
}
pub fn user_documents(kv_handle: &KvHandle, user_id: i32) -> Result<Vec<Document>> {
let prefix = format!("{}:{}:", user_id, ModelType::Document);
let mut ids = vec![];
for row in kv_handle.permissions.prefix(prefix).into_iter() {
let (_key, value) = row?;
let permission: ModelPermission = bincode::deserialize(&value)?;
ids.push(permission.model_id);
}
let documents: Vec<Document> = ids
.into_iter()
.filter_map(|id| Document::load(kv_handle, id).ok().flatten())
.collect();
Ok(documents)
}
}

View file

@ -1,5 +1,6 @@
pub use crate::context::Context; pub use crate::context::Context;
pub use crate::entity::prelude::*; pub use crate::entity::prelude::*;
pub use crate::models::Model;
pub use axum::extract::State; pub use axum::extract::State;
pub use axum::response::{Html, IntoResponse, Response}; pub use axum::response::{Html, IntoResponse, Response};
pub use minijinja::context; pub use minijinja::context;

View file

@ -10,7 +10,7 @@ use tower_sessions::SessionManagerLayer;
use tower_sessions_sqlx_store::{sqlx::SqlitePool, SqliteStore}; use tower_sessions_sqlx_store::{sqlx::SqlitePool, SqliteStore};
use tracing::Level; use tracing::Level;
use crate::{config::CommandLineOptions, context::Context, handler::{home::home_page, login::logout, login_page, login_submit, projects::{create_project, projects_page}}, logging::setup_logging, templates::make_template_loader}; use crate::{config::CommandLineOptions, context::Context, handler::{home::home_page, login::logout, login_page, login_submit, projects::{create_project_page, create_project_submit, projects_page}}, kv::KvHandle, logging::setup_logging, templates::make_template_loader};
pub async fn run() -> Result<()> { pub async fn run() -> Result<()> {
dotenvy::dotenv()?; dotenvy::dotenv()?;
@ -25,7 +25,10 @@ pub async fn run() -> Result<()> {
let session_layer = create_session_manager_layer().await?; let session_layer = create_session_manager_layer().await?;
let context = Context::new(db, template_loader); // TODO: better name, also make it an option
let kv_handles = KvHandle::open("./kvdata/")?;
let context = Context::new(db, kv_handles, template_loader);
let auth_backend = context.clone(); let auth_backend = context.clone();
let auth_layer = AuthManagerLayerBuilder::new(auth_backend, session_layer.clone()).build(); let auth_layer = AuthManagerLayerBuilder::new(auth_backend, session_layer.clone()).build();
@ -41,7 +44,8 @@ pub async fn run() -> Result<()> {
.route("/login", post(login_submit)) .route("/login", post(login_submit))
.route("/logout", get(logout)) .route("/logout", get(logout))
.route("/projects", get(projects_page)) .route("/projects", get(projects_page))
.route("/projects/new", get(create_project)) .route("/projects/new", get(create_project_page))
.route("/projects/new", post(create_project_submit))
.layer(trace_layer) .layer(trace_layer)
.layer(session_layer) .layer(session_layer)
.layer(auth_layer) .layer(auth_layer)

View file

@ -16,13 +16,27 @@
</div> </div>
</div> </div>
<div class="px-8 py-8 flex flex-col gap-y-4"> <form action="/projects/new" method="POST">
<label class="input input-bordered flex items-center gap-2"> <div class="px-8 py-8 flex flex-col gap-y-4">
Name <label class="input input-bordered flex items-center gap-2">
<input type="text" class="grow" placeholder="My New Project" /> Name
</label> <input type="text" id="name" name="name" class="grow" placeholder="My project is called..." />
</label>
<label class="input input-bordered flex items-center gap-2">
Key
<input type="text" id="key" name="key" class="grow" placeholder="Short display code, like BUG" />
</label>
<label class="input input-bordered flex items-center gap-2">
Description
<input type="text" id="description" name="description" class="grow" placeholder="What it's all about" />
</label>
<button class="btn btn-primary">Create</button>
</div>
</form>
</div>
</main> </main>