diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b038ad..c137794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Since this is not a library, this changelog focuses on the changes that are rele - Added GeoIP header mappings and presets for Akamai, Cloudflare, CloudFront, Netlify, and Vercel, with MaxMind results taking precedence when available - Tracker requests now avoid CORS preflight requests by sending JSON as `text/plain` and the event API accepts tracker JSON regardless of content type - Added external authentication with OpenID Connect, Google, and Microsoft Entra ID, including optional user creation and provider access restrictions +- Improved visit duration accuracy by tracking page exits ### Security diff --git a/Cargo.lock b/Cargo.lock index 2da6301..df5da82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4291,6 +4291,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", + "slab", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 8199b37..9b577ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ arc-swap = "1.9" futures-lite = { version = "2.6", default-features = false, features = ["alloc"] } quick_cache = { version = "0.7" } tokio = { version = "1.53", default-features = false, features = ["macros", "rt-multi-thread", "signal", "sync"] } -tokio-util = { version = "0.7", features = ["io"] } +tokio-util = { version = "0.7", features = ["io", "time"] } # encoding hex = { version = "0.4" } diff --git a/README.md b/README.md index 31ba643..1e72781 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@

- liwan.dev - Easy & Privacy-First Web Analytics + liwan.dev - Self-hosted, privacy-first web analytics

-
+
![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/explodingcamera/liwan/test.yaml?style=flat-square) ![GitHub Release](https://img.shields.io/github/v/release/explodingcamera/liwan?style=flat-square) @@ -22,18 +22,24 @@ ## Features -- **Quick setup**\ - Quickly get started with Liwan with a single, self-contained binary . No database or complex setup required. The tracking script is a single line of code that works with any website and less than 1KB in size. -- **Privacy first**\ - Liwan respects your users’ privacy by default. No cookies, no cross-site tracking, no persistent identifiers. All data is stored on your server. -- **Lightweight**\ - You can run Liwan on a cheap VPS, your old mac mini, or even a Raspberry Pi. Written in Rust and using tokio for async I/O, Liwan is fast and efficient. -- **Open source**\ - Fully open source. You can change, extend, and contribute to the codebase. -- **Accurate data**\ - Get accurate data about your website’s visitors, page views, referrers, and more. Liwan detects bots and crawlers and filters them out by default. -- **Real-time analytics**\ - See your website’s traffic in real-time. Liwan updates the dashboard automatically as new visitors come in. +**Understand your traffic**\ +See your most-visited pages, where visitors come from, and how traffic changes over time. The dashboard updates automatically, with bot filtering enabled by default. + +**Easy to self-host**\ +Run Liwan as a single binary or Docker container. The dashboard and database are built in, with no additional services to manage. + +**Privacy first**\ +No tracking cookies or cross-site tracking. Your analytics data stays on your server, and you control what’s collected and how long it’s kept. +[Read more about data collection](https://liwan.dev/collected-data/). + +**Collect only what you need**\ +Choose what data to collect and how long to keep it. Location detail is adjustable, and campaign attribution and session metrics can be disabled independently. + +**Lightweight tracking**\ +Add a small tracking script to your website with a single line of HTML. Works with any framework or CMS. + +**Single sign-on**\ +Manage accounts with Google Workspaces, Microsoft Entra, or your own OpenID Connect provider like Keycloak or Dex. ## License diff --git a/src/app/core/events.rs b/src/app/core/events.rs index fd8585f..651439d 100644 --- a/src/app/core/events.rs +++ b/src/app/core/events.rs @@ -4,13 +4,17 @@ use anyhow::{Context, Result}; use arc_swap::ArcSwap; use chrono::{DateTime, Local, NaiveTime, TimeZone, Utc}; use duckdb::{Connection, Result as DuckResult, params}; +use futures_lite::{StreamExt, future}; use rand::distr::{SampleString, StandardUniform}; use tokio::sync::mpsc::Receiver; +use tokio_util::time::DelayQueue; -use crate::app::models::{Event, GeoDetail, ResolvedCollectionSettings, event_params}; +use crate::app::models::{Event, EventExit, GeoDetail, ResolvedCollectionSettings, event_params}; use crate::app::{DuckDBPool, SqlitePool}; use crate::utils::duckdb::{ParamVec, repeat_vars}; +const EVENT_EXIT_DELAY: std::time::Duration = std::time::Duration::from_secs(30); + #[derive(Clone)] pub struct LiwanEvents { duckdb: DuckDBPool, @@ -97,6 +101,41 @@ impl LiwanEvents { Ok(()) } + /// Process exit updates after a short delay so queued events can reach shared storage first. + pub async fn process_exits(&self, mut exits_rx: Receiver) -> Result<()> { + let mut pending = DelayQueue::new(); + let mut channel_closed = false; + + loop { + if channel_closed && pending.is_empty() { + tracing::info!("Event exit channel closed, stopping event exit processing"); + return Ok(()); + } + + tokio::select! { + exit = exits_rx.recv(), if !channel_closed => match exit { + Some(exit) => { + pending.insert(exit, EVENT_EXIT_DELAY); + } + None => channel_closed = true, + }, + Some(expired) = pending.next(), if !pending.is_empty() => { + let mut exits = vec![expired.into_inner()]; + while let Some(Some(expired)) = future::poll_once(pending.next()).await { + exits.push(expired.into_inner()); + } + + let count = exits.len(); + let events = self.clone(); + match tokio::task::spawn_blocking(move || events.update_exits(exits)).await? { + Ok(matched) => tracing::debug!(count, matched, "Processed event exits"), + Err(err) => tracing::error!(?err, "Failed to process event exits"), + } + } + } + } + } + fn process_events_sync(&self, mut events: Receiver) -> Result<()> { let mut buffer = Vec::with_capacity(1024); let conn = self.duckdb.clone(); @@ -139,6 +178,15 @@ impl LiwanEvents { } } + fn update_exits(&self, exits: Vec) -> Result { + let conn = self.duckdb.get().context("Failed to get DuckDB connection")?; + let mut matched = 0; + for exit in exits { + matched += usize::from(update_event_exit(&conn, &exit)?); + } + Ok(matched) + } + /// Preview or apply collection-setting pruning for a single entity pub fn prune_entity( &self, @@ -205,12 +253,14 @@ impl LiwanEvents { } if !settings.track_sessions { - let sql = "entity_id = ? and (time_from_last_event is not null or time_to_next_event is not null)"; + let sql = "entity_id = ? and (time_from_last_event is not null or time_to_next_event is not null or exited_at is not null)"; stats.cleared_session_events = count_rows(&conn, &format!("select count(*) from events where {sql}"), params![entity_id])?; if !dry_run { conn.execute( - &format!("update events set time_from_last_event = null, time_to_next_event = null where {sql}"), + &format!( + "update events set time_from_last_event = null, time_to_next_event = null, exited_at = null where {sql}" + ), params![entity_id], )?; } @@ -239,6 +289,47 @@ fn count_rows(conn: &Connection, sql: &str, params: impl duckdb::Params) -> Duck conn.query_row(sql, params, |row| row.get(0)) } +fn update_event_exit(conn: &Connection, exit: &EventExit) -> DuckResult { + let sql = "--sql + update events + set exited_at = greatest( + coalesce(exited_at, created_at), + least($exited_at::timestamp, created_at + interval '30 minutes') + ) + where + entity_id = $entity_id and + visitor_group_id = $visitor_group_id and + event = $event and + fqdn is not distinct from $fqdn and + path is not distinct from $path and + time_to_next_event is null and + created_at = ( + select max(candidate.created_at) + from events candidate + where + candidate.entity_id = $entity_id and + candidate.visitor_group_id = $visitor_group_id and + candidate.event = $event and + candidate.fqdn is not distinct from $fqdn and + candidate.path is not distinct from $path and + candidate.time_to_next_event is null and + candidate.created_at <= $exited_at::timestamp and + candidate.created_at >= $exited_at::timestamp - interval '30 minutes' + )"; + let updated = conn.execute( + sql, + duckdb::named_params! { + "entity_id": &exit.entity_id, + "visitor_group_id": &exit.visitor_group_id, + "event": &exit.event, + "fqdn": &exit.fqdn, + "path": &exit.path, + "exited_at": exit.created_at, + }, + )?; + Ok(updated > 0) +} + fn update_event_times(conn: &Connection, from_time: DateTime, entities: &[String]) -> DuckResult<()> { if entities.is_empty() { return Ok(()); @@ -281,3 +372,71 @@ fn update_event_times(conn: &Connection, from_time: DateTime, entities: &[S conn.execute(&sql, duckdb::params_from_iter(params))?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::app::Liwan; + use crate::config::Config; + + fn event(created_at: DateTime) -> Event { + Event { + entity_id: "entity-1".to_string(), + visitor_group_id: "visitor-1".to_string(), + event: "pageview".to_string(), + created_at, + fqdn: Some("example.com".to_string()), + path: Some("/docs".to_string()), + referrer: None, + platform: None, + browser: None, + mobile: None, + country: None, + city: None, + utm_source: None, + utm_medium: None, + utm_campaign: None, + utm_content: None, + utm_term: None, + screen_width: None, + orientation: None, + track_sessions: true, + } + } + + #[test] + fn exit_updates_only_the_latest_event_without_a_next_event() { + let app = Liwan::new_memory(Config::default()).expect("failed to create app"); + let first = Utc::now() - chrono::Duration::minutes(2); + let second = first + chrono::Duration::minutes(1); + app.events.append(vec![event(first), event(second)].into_iter()).expect("failed to append events"); + + let conn = app.events_conn().expect("failed to get event connection"); + let matched = update_event_exit( + &conn, + &EventExit { + entity_id: "entity-1".to_string(), + visitor_group_id: "visitor-1".to_string(), + event: "pageview".to_string(), + created_at: second + chrono::Duration::seconds(15), + fqdn: Some("example.com".to_string()), + path: Some("/docs".to_string()), + }, + ) + .expect("failed to update exit"); + + assert!(matched); + let rows = conn + .prepare("select exited_at from events order by created_at") + .expect("failed to prepare query") + .query_map([], |row| row.get::<_, Option>>(0)) + .expect("failed to query exits") + .collect::, _>>() + .expect("failed to collect exits"); + assert_eq!( + rows[1].map(|value| value.timestamp_millis()), + Some((second + chrono::Duration::seconds(15)).timestamp_millis()) + ); + assert_eq!(rows[0], None); + } +} diff --git a/src/app/core/reports/dimension.rs b/src/app/core/reports/dimension.rs index ef4064b..4ecaee4 100644 --- a/src/app/core/reports/dimension.rs +++ b/src/app/core/reports/dimension.rs @@ -75,7 +75,8 @@ pub fn dimension_report( visitor_group_id, created_at, time_from_last_event, - time_to_next_event + time_to_next_event, + exited_at from events sd where sd.event = ?::text and diff --git a/src/app/core/reports/graph.rs b/src/app/core/reports/graph.rs index 60d96f0..9400505 100644 --- a/src/app/core/reports/graph.rs +++ b/src/app/core/reports/graph.rs @@ -167,7 +167,8 @@ pub fn overall_report( e.visitor_group_id, e.created_at, e.time_from_last_event, - e.time_to_next_event + e.time_to_next_event, + e.exited_at from events e where e.event = ?::text and @@ -181,7 +182,8 @@ pub fn overall_report( sd.visitor_group_id, sd.created_at, sd.time_from_last_event, - sd.time_to_next_event + sd.time_to_next_event, + sd.exited_at from (select * from session_data order by created_at) sd asof join (select * from time_bins order by bin_start) tb on sd.created_at >= tb.bin_start diff --git a/src/app/core/reports/shared.rs b/src/app/core/reports/shared.rs index 48d8640..20ad8e5 100644 --- a/src/app/core/reports/shared.rs +++ b/src/app/core/reports/shared.rs @@ -110,7 +110,12 @@ pub(super) fn metric_aggregate_sql(metric: Metric, alias: &str) -> String { Metric::AvgTimeOnSite => { format!( "--sql - coalesce(avg(extract(epoch from {alias}.time_to_next_event)) filter (where {alias}.time_to_next_event is not null and {alias}.time_to_next_event <= {SESSION_DURATION_SQL}), 0)" + coalesce(avg(extract(epoch from case + when {alias}.time_to_next_event is not null and {alias}.time_to_next_event between interval '0 seconds' and {SESSION_DURATION_SQL} + then {alias}.time_to_next_event + when {alias}.exited_at between {alias}.created_at and {alias}.created_at + {SESSION_DURATION_SQL} + then {alias}.exited_at - {alias}.created_at + end)), 0)" ) } } diff --git a/src/app/core/reports/stats.rs b/src/app/core/reports/stats.rs index fa25c3c..6fd4a1c 100644 --- a/src/app/core/reports/stats.rs +++ b/src/app/core/reports/stats.rs @@ -84,7 +84,8 @@ pub fn overall_stats( e.visitor_group_id, e.created_at, e.time_from_last_event, - e.time_to_next_event + e.time_to_next_event, + e.exited_at from events e where e.event = ?::text and @@ -113,3 +114,30 @@ pub fn overall_stats( Ok(result) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::app::Liwan; + use crate::config::Config; + use chrono::Duration; + + #[test] + fn overall_stats_uses_terminal_event_exit_duration() { + let app = Liwan::new_memory(Config::default()).expect("failed to create app"); + let created_at = Utc::now() - Duration::minutes(5); + let exited_at = created_at + Duration::seconds(90); + let conn = app.events_conn().expect("failed to get event connection"); + conn.execute( + "insert into events (entity_id, visitor_group_id, event, created_at, fqdn, path, exited_at) values (?, ?, ?, ?, ?, ?, ?)", + duckdb::params!["entity-1", "visitor-1", "pageview", created_at, "example.com", "/", exited_at], + ) + .expect("failed to insert event"); + + let range = DateRange { start: created_at - Duration::minutes(1), end: exited_at + Duration::minutes(1) }; + let stats = + overall_stats(&conn, &["entity-1".to_string()], "pageview", &range, &[]).expect("failed to build stats"); + + assert_eq!(stats.avg_time_on_site, Some(90.0)); + } +} diff --git a/src/app/models.rs b/src/app/models.rs index 139749f..398b573 100644 --- a/src/app/models.rs +++ b/src/app/models.rs @@ -29,6 +29,17 @@ pub struct Event { pub track_sessions: bool, } +/// Identifies a stored event that should receive an exit timestamp. +#[derive(Debug, Clone)] +pub struct EventExit { + pub entity_id: String, + pub visitor_group_id: String, + pub event: String, + pub created_at: DateTime, + pub fqdn: Option, + pub path: Option, +} + #[derive(Debug, Clone)] pub struct Project { pub id: String, @@ -457,6 +468,7 @@ macro_rules! event_params { None::, $event.screen_width, $event.orientation, + None::>, ] }; } diff --git a/src/cli.rs b/src/cli.rs index 2bd7e06..53ab693 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -212,7 +212,8 @@ pub fn handle_command(mut config: Config, cmd: Command) -> Result<()> { DevCommand::GenerateOpenApi(_) => { let app = Liwan::try_new(config)?; let (events, _) = tokio::sync::mpsc::channel(1); - let (_, spec) = crate::web::router(app, events)?; + let (exits, _) = tokio::sync::mpsc::channel(1); + let (_, spec) = crate::web::router(app, crate::web::EventQueues { events, exits })?; crate::web::save_spec(spec)?; println!("OpenAPI definition generated"); } diff --git a/src/main.rs b/src/main.rs index acda3a8..2f59ab5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,10 @@ #![forbid(unsafe_code)] use anyhow::Result; -use liwan::app::{Liwan, models::Event}; +use liwan::app::{ + Liwan, + models::{Event, EventExit}, +}; use liwan::{cli, config::Config, web}; use tracing_subscriber::EnvFilter; @@ -16,7 +19,9 @@ async fn main() -> Result<()> { setup_logger(args.log_level)?; let config = Config::load(args.config, std::env::vars())?; - let (s, r) = tokio::sync::mpsc::channel::(1024 * 10); + let (events_tx, events_rx) = tokio::sync::mpsc::channel::(1024 * 10); + let (exits_tx, exits_rx) = tokio::sync::mpsc::channel::(1024 * 10); + let queues = web::EventQueues { events: events_tx, exits: exits_tx }; if let Some(cmd) = args.cmd { return cli::handle_command(config, cmd); @@ -29,8 +34,9 @@ async fn main() -> Result<()> { tokio::select! { biased; _ = liwan::utils::signals::shutdown() => app_copy.shutdown(), - res = web::start_webserver(app.clone(), s) => res, - res = app.events.process_events(r) => res, + res = web::start_webserver(app.clone(), queues) => res, + res = app.events.process_events(events_rx) => res, + res = app.events.process_exits(exits_rx) => res, } } diff --git a/src/migrations/events/V7__event_exits.sql b/src/migrations/events/V7__event_exits.sql new file mode 100644 index 0000000..dae4f26 --- /dev/null +++ b/src/migrations/events/V7__event_exits.sql @@ -0,0 +1 @@ +alter table events add column exited_at timestamp; diff --git a/src/web/mod.rs b/src/web/mod.rs index eee58b4..541cb44 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -21,7 +21,10 @@ use tower_http::{ timeout::RequestBodyDeadlineLayer, }; -use crate::app::{Liwan, models::Event}; +use crate::app::{ + Liwan, + models::{Event, EventExit}, +}; use crate::web::webext::serve; pub use session::MaybeSessionId; @@ -39,9 +42,19 @@ struct Script; pub struct RouterState { pub app: Arc, pub events: Sender, + pub exits: Sender, pub report_permits: Arc, } +/// Event ingestion queues used by the web server. +#[derive(Clone)] +pub struct EventQueues { + /// Queue for normal event inserts. + pub events: Sender, + /// Queue for delayed event exit updates. + pub exits: Sender, +} + // feTS treats directly resolved component references as circular and falls back to less precise types. #[derive(Clone)] struct WrapSchemaRefs; @@ -76,7 +89,7 @@ impl Deref for RouterState { } } -pub fn router(app: Arc, events: Sender) -> Result<(axum::Router<()>, openapi::OpenApi)> { +pub fn router(app: Arc, queues: EventQueues) -> Result<(axum::Router<()>, openapi::OpenApi)> { aide::generate::in_context(|ctx| { ctx.schema = ctx.schema.settings().clone().with_transform(WrapSchemaRefs).into_generator(); }); @@ -139,7 +152,8 @@ pub fn router(app: Arc, events: Sender) -> Result<(axum::Router<() .layer(set_headers) .with_state(RouterState { app: app.clone(), - events, + events: queues.events, + exits: queues.exits, report_permits: Arc::new(Semaphore::new(app.config.limits.report_max_concurrency)), }) .finish_api(&mut api); @@ -168,7 +182,7 @@ pub fn save_spec(spec: openapi::OpenApi) -> Result<()> { Ok(()) } -pub async fn start_webserver(app: Arc, events: Sender) -> Result<()> { +pub async fn start_webserver(app: Arc, queues: EventQueues) -> Result<()> { match app.onboarding.token() { Some(onboarding) => { let get_started = format!("{}/setup?t={}", app.config.base_url, onboarding); @@ -181,7 +195,7 @@ pub async fn start_webserver(app: Arc, events: Sender) -> Result<( } } - let router = router(app.clone(), events)?; + let router = router(app.clone(), queues)?; #[cfg(debug_assertions)] save_spec(router.1)?; diff --git a/src/web/routes/event.rs b/src/web/routes/event.rs index 13c059d..f8a8b9e 100644 --- a/src/web/routes/event.rs +++ b/src/web/routes/event.rs @@ -1,5 +1,6 @@ use crate::app::models::{ - FilterType, GeoDetail, IngestDropRule, IngestFilter, ResolvedCollectionSettings, VisitorGroupMode, hostname_allowed, + EventExit, FilterType, GeoDetail, IngestDropRule, IngestFilter, ResolvedCollectionSettings, VisitorGroupMode, + hostname_allowed, }; use crate::app::{Liwan, models::Event}; use crate::config::Config; @@ -53,6 +54,8 @@ struct EventRequest { referrer: Option, screen_width: Option, orientation: Option, + #[serde(default)] + exit: bool, } impl EventRequest { @@ -165,11 +168,24 @@ async fn event_handler( .http_status(StatusCode::INTERNAL_SERVER_ERROR)?; match res { - Ok(Some(event)) => { + Ok(Some((event, false))) => { if events.send_timeout(event, std::time::Duration::from_secs(2)).await.is_err() { tracing::warn!("Failed to send event, channel full"); } } + Ok(Some((event, true))) => { + let exit = EventExit { + entity_id: event.entity_id, + visitor_group_id: event.visitor_group_id, + event: event.event, + created_at: event.created_at, + fqdn: event.fqdn, + path: event.path, + }; + if state.exits.send_timeout(exit, std::time::Duration::from_secs(2)).await.is_err() { + tracing::warn!("Failed to send event exit, channel full"); + } + } // event was filtered out, do nothing Ok(None) => {} Err(e) => tracing::warn!("Failed to process event: {:?}", e), @@ -185,7 +201,8 @@ fn process_event( ip: Option, geo_headers: GeoLocationHeaders, user_agent: headers::UserAgent, -) -> Result> { +) -> Result> { + let is_exit = event.exit; let referrer = match process_referer(event.referrer.as_deref()) { Referrer::Fqdn(fqdn) => Some(fqdn), Referrer::Unknown(r) => r, @@ -215,6 +232,14 @@ fn process_event( return Ok(None); } + if is_exit + && (!settings.track_sessions + || settings.visitor_group_mode == VisitorGroupMode::RandomPerRequest + || ip.is_none()) + { + return Ok(None); + } + let visitor_group_id = resolve_visitor_group_id(&settings, ip, user_agent.as_str(), &app.events.get_salt()?, &event.entity_id); @@ -272,7 +297,7 @@ fn process_event( return Ok(None); } - Ok(Some(event)) + Ok(Some((event, is_exit))) } fn ingest_drop_rule_matches(event: &Event, rule: &IngestDropRule) -> bool { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 7c34b07..520fe5a 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,18 +3,28 @@ use axum_test::TestServer; use cookie::Cookie; use liwan::{ - app::{Liwan, models::Event}, + app::{ + Liwan, + models::{Event, EventExit}, + }, config::Config, }; use serde_json::json; use std::sync::Arc; +pub struct EventReceivers { + pub events: tokio::sync::mpsc::Receiver, + pub exits: tokio::sync::mpsc::Receiver, +} + pub fn app() -> std::sync::Arc { Liwan::new_memory(Config::default()).unwrap() } -pub fn events() -> (tokio::sync::mpsc::Sender, tokio::sync::mpsc::Receiver) { - tokio::sync::mpsc::channel::(1024 * 10) +pub fn events() -> (liwan::web::EventQueues, EventReceivers) { + let (events, event_rx) = tokio::sync::mpsc::channel::(1024 * 10); + let (exits, exit_rx) = tokio::sync::mpsc::channel::(1024 * 10); + (liwan::web::EventQueues { events, exits }, EventReceivers { events: event_rx, exits: exit_rx }) } pub struct TestClient { @@ -22,8 +32,8 @@ pub struct TestClient { } impl TestClient { - pub fn new(app: Arc, events: tokio::sync::mpsc::Sender) -> Self { - let (router, _) = liwan::web::router(app, events).unwrap(); + pub fn new(app: Arc, queues: liwan::web::EventQueues) -> Self { + let (router, _) = liwan::web::router(app, queues).unwrap(); let server = TestServer::new(router.into_make_service_with_connect_info::()); Self { server } } diff --git a/tests/event.rs b/tests/event.rs index b1a6b0a..40c8dd2 100644 --- a/tests/event.rs +++ b/tests/event.rs @@ -1,6 +1,8 @@ mod common; use anyhow::Result; use liwan::app::models::Entity; +use liwan::config::Config; +use liwan::utils::ip_headers::{ClientIpHeaderSource, TrustedProxy}; use serde_json::json; #[tokio::test] @@ -24,7 +26,7 @@ async fn test_event() -> Result<()> { let res = client.post_with_headers("/api/event", event, vec![("user-agent".to_string(), "test".to_string())]).await; res.assert_status_success(); - let event = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv()) + let event = tokio::time::timeout(std::time::Duration::from_secs(1), rx.events.recv()) .await .expect("event should be received") .expect("event channel should not be closed"); @@ -99,11 +101,47 @@ async fn deleted_entity_does_not_accept_events() -> Result<()> { let headers = vec![("user-agent".to_string(), "test".to_string())]; client.post_with_headers("/api/event", event.clone(), headers.clone()).await.assert_status_success(); - rx.recv().await.expect("event should be received"); + rx.events.recv().await.expect("event should be received"); app.entities.delete("entity-to-delete")?; client.post_with_headers("/api/event", event, headers).await.assert_status_success(); - assert!(rx.try_recv().is_err(), "deleted entity should not produce an event"); + assert!(rx.events.try_recv().is_err(), "deleted entity should not produce an event"); + + Ok(()) +} + +#[tokio::test] +async fn exit_payload_uses_the_exit_queue() -> Result<()> { + let mut config = Config::default(); + config.client_ip_headers = vec![ClientIpHeaderSource::Header("x-client-ip".to_string())].into(); + config.trusted_proxies = vec![TrustedProxy::Ip("127.0.0.1".parse()?)].into(); + let app = liwan::app::Liwan::new_memory(config)?; + let (queues, mut receivers) = common::events(); + let client = common::TestClient::new(app.clone(), queues); + app.seed_database(0)?; + + let event = json!({ + "entity_id": "entity-1", + "name": "pageview", + "url": "https://example.com/", + "exit": true + }); + client + .post_with_headers( + "/api/event", + event, + vec![("user-agent".to_string(), "test".to_string()), ("x-client-ip".to_string(), "8.8.8.8".to_string())], + ) + .await + .assert_status_success(); + + let exit = tokio::time::timeout(std::time::Duration::from_secs(1), receivers.exits.recv()) + .await + .expect("exit should be received") + .expect("exit channel should not be closed"); + assert_eq!(exit.event, "pageview"); + assert_eq!(exit.fqdn.as_deref(), Some("example.com")); + assert!(receivers.events.try_recv().is_err(), "exit payload should not insert an event"); Ok(()) } diff --git a/tracker/README.md b/tracker/README.md index 6e70aaa..064ac5e 100644 --- a/tracker/README.md +++ b/tracker/README.md @@ -19,6 +19,8 @@ When the script is loaded directly in the browser, it will automatically send pa > ``` +Pageviews send a best-effort exit signal when the page becomes hidden. Set `data-exit="false"` to disable exit tracking. + When using the npm package, call `trackPageviews()` to start automatic pageview tracking: ```ts @@ -76,6 +78,14 @@ export type EventOptions = { * Required for custom events. */ entity?: string; + + /** + * Whether this event should be reported again when the page becomes hidden. + * + * Defaults to `true` for pageviews and `false` for other events. This option + * is ignored in server-side environments. + */ + exit?: boolean; }; /** diff --git a/tracker/script.d.ts b/tracker/script.d.ts index 2b43cf6..7cd068e 100644 --- a/tracker/script.d.ts +++ b/tracker/script.d.ts @@ -30,11 +30,17 @@ export type EventOptions = { * Required for custom events. */ entity?: string; + /** + * Whether this event should be reported again when the page becomes hidden. + * + * Defaults to `true` for pageviews and `false` for other events. This option is ignored in server-side environments. + */ + exit?: boolean; }; /** * Sends an event to the Liwan API. * - * @param name The name of the event. Defaults to "pageview". Currencly, custom event names are not supported and will be treated as "pageview". + * @param name The name of the event. Defaults to "pageview". * @param options Additional options for the event. See {@link EventOptions}. * @returns A promise that resolves when the event has been sent * @throws If {@link EventOptions.endpoint} is not provided in server-side environments. diff --git a/tracker/script.min.js b/tracker/script.min.js index 2e79671..ec353a5 100644 --- a/tracker/script.min.js +++ b/tracker/script.min.js @@ -1 +1 @@ -let o=null,d=null,u=null,g=null;const i=typeof window>"u";typeof document<"u"&&(o=document.querySelector(`script[src^="${import.meta.url}"]`)??document.querySelector("script:not([src])[data-api][data-entity]"),d=o?.getAttribute("data-api")||o?.src&&`${new URL(o.src).origin}/api/event`||null,u=o?.getAttribute("data-entity")||null,g=document.referrer);const p=t=>console.info(`[liwan]: ${t}`),f=t=>p(`Ignoring event: ${t}`),l=t=>{throw new Error(`Failed to send event: ${t}`)},m=["utm_campaign","utm_content","utm_medium","utm_source","utm_term","campaign","content","medium","source","term","ref","referrer","referer"],w=t=>{const n=i?new URL(t):new URL(t,location.href),r=new URLSearchParams;for(const[e,a]of n.searchParams)m.includes(e)&&r.append(e,a);return n.search=r.toString(),n.hash="",n.toString()};async function h(t="pageview",n){const r=n?.endpoint||d;if(!r)return l("endpoint is required");if(!i&&localStorage?.getItem("disable-liwan"))return f("localStorage flag");if(!i&&(/^localhost$|^127(?:\.\d+){0,2}\.\d+$|^(?:\[::1\]|::1)$/.test(location.hostname)||location.protocol==="file:"))return f("localhost");const e=i?void 0:window.screen?.width,a=e==null?void 0:e<480?"xs":e<768?"sm":e<1024?"md":e<1280?"lg":e<1536?"xl":"2xl",c=n?.url||(i?null:location.href);if(!c)return l("url is required");const s=await fetch(r,{method:"POST",headers:{"Content-Type":"text/plain;charset=UTF-8"},keepalive:!0,body:JSON.stringify({name:t,entity_id:n?.entity||u,referrer:n?.referrer||g,url:w(c),screen_width:a,orientation:i?void 0:window.screen.orientation?.type.startsWith("portrait")?"portrait":"landscape"})});s.ok||l(`${s.status} ${s.statusText}`.trim())}const y=t=>{window.__liwan_loaded=!0;let n;const r=()=>{n!==location.pathname&&(n=location.pathname,h("pageview",t).catch(e=>p(e instanceof Error?e.message:String(e))))};window.navigation?window.navigation.addEventListener("currententrychange",()=>r()):window.addEventListener("popstate",()=>r()),r()};!i&&!window.__liwan_loaded&&o&&y();export{h as event,y as trackPageviews}; +let a=null,f=null,m=null,w=null,y=!0;const i=typeof window>"u";let o=null,h=!1,l=!1;typeof document<"u"&&(a=document.querySelector(`script[src^="${import.meta.url}"]`)??document.querySelector("script:not([src])[data-api][data-entity]"),f=a?.getAttribute("data-api")||a?.src&&`${new URL(a.src).origin}/api/event`||null,m=a?.getAttribute("data-entity")||null,w=document.referrer,y=a?.getAttribute("data-exit")!=="false");const c=t=>console.info(`[liwan]: ${t}`),v=t=>c(`Ignoring event: ${t}`),u=t=>{throw new Error(`Failed to send event: ${t}`)},_=()=>{if(i||document.visibilityState!=="hidden"||!o||l)return;l=!0;const t=JSON.stringify({...o.payload,exit:!0});fetch(o.endpoint,{method:"POST",headers:{"Content-Type":"text/plain;charset=UTF-8"},keepalive:!0,body:t}).catch(e=>c(e instanceof Error?e.message:String(e)))},x=()=>{h||i||(h=!0,document.addEventListener("visibilitychange",()=>{if(document.visibilityState!=="hidden"){l=!1;return}_()}))},S=["utm_campaign","utm_content","utm_medium","utm_source","utm_term","campaign","content","medium","source","term","ref","referrer","referer"],E=t=>{const e=i?new URL(t):new URL(t,location.href),r=new URLSearchParams;for(const[n,s]of e.searchParams)S.includes(n)&&r.append(n,s);return e.search=r.toString(),e.hash="",e.toString()};async function b(t="pageview",e){const r=e?.endpoint||f;if(!r)return u("endpoint is required");if(!i&&localStorage?.getItem("disable-liwan"))return v("localStorage flag");if(!i&&(/^localhost$|^127(?:\.\d+){0,2}\.\d+$|^(?:\[::1\]|::1)$/.test(location.hostname)||location.protocol==="file:"))return v("localhost");const n=i?void 0:window.screen?.width,s=n==null?void 0:n<480?"xs":n<768?"sm":n<1024?"md":n<1280?"lg":n<1536?"xl":"2xl",g=e?.url||(i?null:location.href);if(!g)return u("url is required");const p={name:t,entity_id:e?.entity||m,referrer:e?.referrer||w,url:E(g),screen_width:s,orientation:i?void 0:window.screen.orientation?.type.startsWith("portrait")?"portrait":"landscape"},d=await fetch(r,{method:"POST",headers:{"Content-Type":"text/plain;charset=UTF-8"},keepalive:!0,body:JSON.stringify(p)});d.ok||u(`${d.status} ${d.statusText}`.trim()),!i&&(e?.exit??(t==="pageview"&&y))?(o={endpoint:r,payload:p},l=!1,x(),_()):!i&&t==="pageview"&&(o=null)}const T=t=>{window.__liwan_loaded=!0;let e;const r=()=>{e!==location.pathname&&(e=location.pathname,b("pageview",t).catch(n=>c(n instanceof Error?n.message:String(n))))};window.navigation?window.navigation.addEventListener("currententrychange",()=>r()):window.addEventListener("popstate",()=>r()),r()};!i&&!window.__liwan_loaded&&a&&T();export{b as event,T as trackPageviews}; diff --git a/tracker/script.ts b/tracker/script.ts index b765abd..7a4ef61 100644 --- a/tracker/script.ts +++ b/tracker/script.ts @@ -11,6 +11,7 @@ type Payload = { referrer?: string; screen_width?: string; orientation?: string; + exit?: boolean; // biome-ignore lint/suspicious/noExplicitAny: we want to allow any additional properties to be sent in the payload } & Record; @@ -44,13 +45,24 @@ export type EventOptions = { * Required for custom events. */ entity?: string; + + /** + * Whether this event should be reported again when the page becomes hidden. + * + * Defaults to `true` for pageviews and `false` for other events. This option is ignored in server-side environments. + */ + exit?: boolean; }; let scriptEl: HTMLScriptElement | null = null; let endpoint: string | null = null; let entity: string | null = null; let referrer: string | null = null; +let defaultExit = true; const noWindow = typeof window === "undefined"; +let currentExit: { endpoint: string; payload: Payload } | null = null; +let exitListenerInstalled = false; +let exitSentWhileHidden = false; if (typeof document !== "undefined") { scriptEl = @@ -62,6 +74,7 @@ if (typeof document !== "undefined") { entity = scriptEl?.getAttribute("data-entity") || null; referrer = document.referrer; + defaultExit = scriptEl?.getAttribute("data-exit") !== "false"; } const log = (message: string) => console.info(`[liwan]: ${message}`); @@ -70,6 +83,32 @@ const reject = (message: string) => { throw new Error(`Failed to send event: ${message}`); }; +const sendCurrentExit = () => { + if (noWindow || document.visibilityState !== "hidden" || !currentExit || exitSentWhileHidden) return; + exitSentWhileHidden = true; + + const body = JSON.stringify({ ...currentExit.payload, exit: true }); + void fetch(currentExit.endpoint, { + method: "POST", + headers: { "Content-Type": "text/plain;charset=UTF-8" }, + keepalive: true, + body, + }).catch((error) => log(error instanceof Error ? error.message : String(error))); +}; + +const installExitListener = () => { + if (exitListenerInstalled || noWindow) return; + exitListenerInstalled = true; + + document.addEventListener("visibilitychange", () => { + if (document.visibilityState !== "hidden") { + exitSentWhileHidden = false; + return; + } + sendCurrentExit(); + }); +}; + const ATTRIBUTION_QUERY_PARAMS = [ "utm_campaign", "utm_content", @@ -104,7 +143,7 @@ const sanitizeUrl = (value: string) => { /** * Sends an event to the Liwan API. * - * @param name The name of the event. Defaults to "pageview". Currencly, custom event names are not supported and will be treated as "pageview". + * @param name The name of the event. Defaults to "pageview". * @param options Additional options for the event. See {@link EventOptions}. * @returns A promise that resolves when the event has been sent * @throws If {@link EventOptions.endpoint} is not provided in server-side environments. @@ -144,27 +183,39 @@ export async function event(name: string = "pageview", options?: EventOptions): const url = options?.url || (!noWindow ? location.href : null); if (!url) return reject("url is required"); - const response = await fetch(endpoint_url, { + const payload = { + name, + entity_id: options?.entity || entity, + referrer: options?.referrer || referrer, + url: sanitizeUrl(url), + screen_width, + orientation: noWindow + ? undefined + : window.screen.orientation?.type.startsWith("portrait") + ? "portrait" + : "landscape", + }; + const request = fetch(endpoint_url, { method: "POST", headers: { "Content-Type": "text/plain;charset=UTF-8" }, // we use text/plain to avoid preflight requests keepalive: true, // allow the request to be sent even if the page is being unloaded - body: JSON.stringify({ - name, - entity_id: options?.entity || entity, - referrer: options?.referrer || referrer, - url: sanitizeUrl(url), - screen_width, - orientation: noWindow - ? undefined - : window.screen.orientation?.type.startsWith("portrait") - ? "portrait" - : "landscape", - }), + body: JSON.stringify(payload), }); + const response = await request; + if (!response.ok) { reject(`${response.status} ${response.statusText}`.trim()); } + + if (!noWindow && (options?.exit ?? (name === "pageview" && defaultExit))) { + currentExit = { endpoint: endpoint_url, payload }; + exitSentWhileHidden = false; + installExitListener(); + sendCurrentExit(); + } else if (!noWindow && name === "pageview") { + currentExit = null; + } } /** diff --git a/web/astro.config.ts b/web/astro.config.ts index 7b0e415..081e065 100644 --- a/web/astro.config.ts +++ b/web/astro.config.ts @@ -56,6 +56,14 @@ export default defineConfig({ styles: ["normal"], subsets: ["latin", "latin-ext"], }, + { + provider: fontProviders.fontsource(), + name: "Google Sans", + cssVariable: "--font-google-sans", + weights: ["400 500 700"], + styles: ["normal"], + subsets: ["latin", "latin-ext"], + }, ], vite: { server: { proxy }, diff --git a/web/src/components/dashboard/project/card.module.css b/web/src/components/dashboard/project/card.module.css index 0bc08a7..0bb4970 100644 --- a/web/src/components/dashboard/project/card.module.css +++ b/web/src/components/dashboard/project/card.module.css @@ -47,7 +47,7 @@ button.card { z-index: 0; margin: 0; - padding: 0.5rem 0.6rem 0.3rem 0.6rem; + padding: 0.55rem 0.6rem 0.25rem; transition: background-color 0.2s ease; border-radius: var(--pico-border-radius); diff --git a/web/src/components/dashboard/project/dimensions/dimensions.module.css b/web/src/components/dashboard/project/dimensions/dimensions.module.css index e10cc32..8cb7e73 100644 --- a/web/src/components/dashboard/project/dimensions/dimensions.module.css +++ b/web/src/components/dashboard/project/dimensions/dimensions.module.css @@ -5,6 +5,10 @@ } .tabs { + display: flex; + flex: 1; + flex-direction: column; + .tabsList { display: flex; gap: 0.5rem; @@ -43,6 +47,7 @@ .tabsContent { display: flex; + flex: 1; flex-direction: column; } } @@ -118,7 +123,8 @@ all: unset; cursor: pointer; color: var(--pico-contrast); - margin-top: 0.2rem; + margin-top: auto; + padding-top: 0.4rem; display: flex; justify-content: center; align-items: center; diff --git a/web/src/components/dashboard/project/index.module.css b/web/src/components/dashboard/project/index.module.css index 696870b..d685a07 100644 --- a/web/src/components/dashboard/project/index.module.css +++ b/web/src/components/dashboard/project/index.module.css @@ -107,7 +107,7 @@ width: calc(50% - 1rem); > div { - padding: 1rem; + padding: var(--pico-block-spacing-vertical) var(--pico-block-spacing-horizontal); padding-left: 0; } } @@ -132,7 +132,7 @@ } .geoTable { - padding: 1rem; + padding: var(--pico-block-spacing-vertical) var(--pico-block-spacing-horizontal); padding-top: 0; > div { diff --git a/web/src/components/dashboard/project/project-header.module.css b/web/src/components/dashboard/project/project-header.module.css index 19d77cc..2b10db7 100644 --- a/web/src/components/dashboard/project/project-header.module.css +++ b/web/src/components/dashboard/project/project-header.module.css @@ -16,7 +16,6 @@ line-height: normal; margin-left: 0.5rem; - transform: translateY(-0.08rem); display: inline-flex; gap: 0.2rem; align-items: center; diff --git a/web/src/components/dashboard/project/project-header.tsx b/web/src/components/dashboard/project/project-header.tsx index 128270b..963c888 100644 --- a/web/src/components/dashboard/project/project-header.tsx +++ b/web/src/components/dashboard/project/project-header.tsx @@ -9,12 +9,10 @@ import { CardLink } from "./card"; export const ProjectHeader = ({ project, stats }: { stats?: StatsResponse; project: ProjectResponse }) => { return (

- - - {project.public ? project.unlisted && : } - {project.displayName} - - + + {project.public ? project.unlisted && : } + {project.displayName} + {stats && }

); diff --git a/web/src/components/settings/auth/authentication.module.css b/web/src/components/settings/auth/authentication.module.css index b291d3a..33a2241 100644 --- a/web/src/components/settings/auth/authentication.module.css +++ b/web/src/components/settings/auth/authentication.module.css @@ -159,15 +159,6 @@ margin: 0; } -.callback { - display: block; - padding: 0.15rem 0; - background: transparent; - color: var(--pico-muted-color); - overflow-wrap: anywhere; - user-select: all; -} - @media (max-width: 600px) { .optionList, .providerGrid { diff --git a/web/src/components/settings/auth/index.tsx b/web/src/components/settings/auth/index.tsx index b0d4a0d..bfe0494 100644 --- a/web/src/components/settings/auth/index.tsx +++ b/web/src/components/settings/auth/index.tsx @@ -5,6 +5,7 @@ import { SiGoogle, SiOpenid } from "@icons-pack/react-simple-icons"; import { KeyRoundIcon } from "lucide-react"; import { api } from "@/api"; +import { CopyableValue } from "@/components/ui/snippet"; import { createToast } from "@/components/ui/toast"; import type { ExternalAuthProvider, ExternalAuthSettings, ExternalAuthSettingsUpdate } from "@/constants"; import { SettingsField, SettingsForm, SettingsHeader, SettingsSwitch } from "../form"; @@ -55,7 +56,7 @@ const ProviderSettings = ({ <>
Sign-in method -

Use Liwan passwords only, or add single sign-on with one external provider.

+

Use liwan passwords only, or add single sign-on with one external provider.

{providers.map((provider) => ( @@ -93,9 +94,7 @@ const ProviderSettings = ({ ))}
{!settings.enabled && ( -

- Liwan username and password sign-in is active. No additional setup is required. -

+

Users sign in with their liwan username and password.

)}
{settings.enabled && ( @@ -321,7 +320,7 @@ export const AuthenticationSettingsPage = () => {

Callback URL

Add this exact URL to the provider application's allowed redirect URLs.

- {settings.callbackUrl} +
)} {error &&
{error}
} diff --git a/web/src/components/settings/collection/collection.module.css b/web/src/components/settings/collection/collection.module.css index 29529b8..0c82a1c 100644 --- a/web/src/components/settings/collection/collection.module.css +++ b/web/src/components/settings/collection/collection.module.css @@ -62,10 +62,9 @@ margin: 0; padding: 0.35rem 0.65rem; border: 0; - border-bottom: 2px solid transparent; border-radius: 0; background: transparent; - color: var(--pico-muted-color); + color: color-mix(in srgb, var(--pico-muted-color) 88%, transparent); box-shadow: none; } @@ -77,8 +76,7 @@ .activeTab, .activeTab:hover { - border-bottom-color: var(--pico-primary); - color: var(--pico-primary); + color: var(--pico-h1-color); } .panel { diff --git a/web/src/components/settings/dialogs.module.css b/web/src/components/settings/dialogs.module.css index 4b11ec4..7740ac4 100644 --- a/web/src/components/settings/dialogs.module.css +++ b/web/src/components/settings/dialogs.module.css @@ -59,29 +59,26 @@ .tab { --pico-background-color: transparent; --pico-border-color: transparent; - --pico-color: var(--pico-muted-color); width: auto; margin: 0; padding: 0.35rem 0.65rem; border: 0; - border-bottom: 2px solid transparent; border-radius: 0; background: transparent; - color: var(--pico-muted-color); + color: color-mix(in srgb, var(--pico-muted-color) 88%, transparent); box-shadow: none; } .tab:hover { background: transparent; - color: var(--pico-h1-color); + color: var(--pico-color); box-shadow: none; } .activeTab, .activeTab:hover, .tab[data-selected] { - border-bottom-color: var(--pico-primary-hover); - color: var(--pico-primary-hover); + color: var(--pico-h1-color); } .tabPanel { diff --git a/web/src/components/settings/form.module.css b/web/src/components/settings/form.module.css index 0c50b22..35ac535 100644 --- a/web/src/components/settings/form.module.css +++ b/web/src/components/settings/form.module.css @@ -10,6 +10,10 @@ margin-bottom: 0.9rem; } +.header[data-has-back] { + margin-bottom: 1.25rem; +} + .titleGroup { min-width: 0; display: flex; @@ -24,6 +28,10 @@ white-space: nowrap; } +.header[data-has-back] .titleGroup h1 { + font-size: 1rem; +} + .backButton { width: 2rem; height: 2rem; @@ -68,28 +76,25 @@ .tab { --pico-background-color: transparent; --pico-border-color: transparent; - --pico-color: var(--pico-muted-color); width: auto; margin: 0; padding: 0.35rem 0.65rem; border: 0; - border-bottom: 2px solid transparent; border-radius: 0; background: transparent; - color: var(--pico-muted-color); + color: color-mix(in srgb, var(--pico-muted-color) 88%, transparent); box-shadow: none; } .tab:hover { background: transparent; - color: var(--pico-h1-color); + color: var(--pico-color); box-shadow: none; } .tab[data-active], .tab[data-selected] { - border-bottom-color: var(--pico-primary-hover); - color: var(--pico-primary-hover); + color: var(--pico-h1-color); } .panel { diff --git a/web/src/components/settings/form.tsx b/web/src/components/settings/form.tsx index f3bdba7..09eab4d 100644 --- a/web/src/components/settings/form.tsx +++ b/web/src/components/settings/form.tsx @@ -38,7 +38,7 @@ export const SettingsHeader = ({ saveForm?: string; }) => ( <> -