Skip to content

Commit 8d54c4d

Browse files
authored
Reduce site deletions impact on ClickHouse (#6591)
* Implement cache broadcasting deletes * Remove domains from Site.Cache upon deletion * Query by domain_changed_from in cache DB fallback * Migration: pending stats deletions * Implement PendingStatsDeletion schema * Implement querying site stats range * Cosmetics: use new_site factory * Implement computing partition_ids out of dates * Interface for storing pending stats deletions * Migration: squash non-nullables * Drop changeset usage for pending deletions * Don't store pending stats deletion when there's nothing to delete * Store pending stats deletion on site removal * Migration: index by reason * List pending deletions by reason * Migration: dates non-nullable * WIP: Clean up sites based on pending deletions * Lightweight delete + by partition where possible c * Better log message * !fixup * Bulk insert oprhaned site ids * s/list_by_reason/list * Clear pending deletions once removal is done * Adjust the cron clock * Add basic telemetry (even though most of it is async exec) * Use dedicated deletion repo not to ever hog ingest * Format * Use UTC dates for partitioning basis * Don't depend on postgres pointers for deletion scope * Credo * bump log level * capture log
1 parent a3fd4b9 commit 8d54c4d

18 files changed

Lines changed: 609 additions & 94 deletions

config/runtime.exs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -705,12 +705,12 @@ config :plausible, Plausible.AsyncInsertRepo,
705705
materialized_views_ignore_errors: 1
706706
]
707707

708-
config :plausible, Plausible.ImportDeletionRepo,
708+
config :plausible, Plausible.DeletionRepo,
709709
queue_target: 500,
710710
queue_interval: 2000,
711711
url: ch_db_url,
712712
transport_opts: ch_transport_opts,
713-
pool_size: 1
713+
pool_size: 2
714714

715715
config :plausible, Plausible.Ingestion.Persistor,
716716
backend: persistor_backend,
@@ -848,10 +848,10 @@ cloud_cron = [
848848
{"0 0 * * *", Plausible.Workers.LockSites},
849849
# Daily at 8
850850
{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil},
851-
# First sunday of the month, 4:00 UTC
852-
{"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites},
853-
# Daily at 4:00 UTC
854-
{"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},
851+
# Every Tuesday, 3:00 UTC
852+
{"0 3 * * TUE", Plausible.Workers.ClickhouseCleanSites},
853+
# Daily at 5:00 UTC
854+
{"0 5 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},
855855
# Daily at 2:00 UTC
856856
{"0 2 * * *", Plausible.Workers.ScoreTrialProspects}
857857
]

lib/plausible/application.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ defmodule Plausible.Application do
4444
Plausible.ClickhouseRepo,
4545
Plausible.IngestRepo,
4646
Plausible.AsyncInsertRepo,
47-
Plausible.ImportDeletionRepo,
47+
Plausible.DeletionRepo,
4848
Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency,
4949
ttl_check_interval: :timer.minutes(5),
5050
n_lock_partitions: 1,

lib/plausible/cache.ex

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,19 @@ defmodule Plausible.Cache do
8383
:ok
8484
end
8585

86+
@spec broadcast_delete(any(), Keyword.t()) :: :ok
87+
def broadcast_delete(key, opts \\ []) do
88+
cache_name = Keyword.get(opts, :cache_name, name())
89+
multicall_timeout = Keyword.get(opts, :multicall_timeout, :timer.seconds(5))
90+
91+
{:ok, _} =
92+
Task.start(fn ->
93+
:rpc.multicall(Adapter, :delete, [cache_name, key], multicall_timeout)
94+
end)
95+
96+
:ok
97+
end
98+
8699
@spec get(any(), Keyword.t()) :: any() | nil
87100
def get(key, opts \\ [])
88101

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
defmodule Plausible.ImportDeletionRepo do
1+
defmodule Plausible.DeletionRepo do
22
@moduledoc """
33
A dedicated repo for import related mutations
44
"""
@@ -9,7 +9,7 @@ defmodule Plausible.ImportDeletionRepo do
99

1010
defmacro __using__(_) do
1111
quote do
12-
alias Plausible.ImportDeletionRepo
12+
alias Plausible.DeletionRepo
1313
import Ecto
1414
import Ecto.Query, only: [from: 1, from: 2]
1515
end
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
defmodule Plausible.PendingStatsDeletion do
2+
@moduledoc """
3+
Schema for tracking pending deletions from ClickHouse
4+
"""
5+
6+
use Ecto.Schema
7+
8+
@reasons [:user_request]
9+
10+
@type t() :: %__MODULE__{}
11+
12+
schema "pending_stats_deletions" do
13+
field :site_id, :integer
14+
field :reason, Ecto.Enum, values: @reasons, default: :user_request
15+
16+
timestamps()
17+
end
18+
end
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
defmodule Plausible.PendingStatsDeletions do
2+
@moduledoc """
3+
Context for pending stats deletions
4+
"""
5+
6+
import Ecto.Query
7+
8+
alias Plausible.ClickhouseRepo
9+
alias Plausible.PendingStatsDeletion
10+
alias Plausible.Repo
11+
alias Plausible.Site
12+
13+
@spec store(Site.t(), atom()) :: {:ok, PendingStatsDeletion.t()}
14+
def store(%Site{} = site, reason \\ :user_request) do
15+
Repo.insert(%PendingStatsDeletion{site_id: site.id, reason: reason})
16+
end
17+
18+
@spec list(atom()) :: [pos_integer()]
19+
def list(reason \\ :user_request) do
20+
from(p in PendingStatsDeletion,
21+
where: p.reason == ^reason,
22+
distinct: true,
23+
order_by: p.site_id,
24+
select: p.site_id
25+
)
26+
|> Repo.all()
27+
end
28+
29+
@spec clear([pos_integer()], atom()) :: {non_neg_integer(), nil}
30+
def clear(site_ids, reason \\ :user_request)
31+
def clear([], _reason), do: {0, nil}
32+
33+
def clear(site_ids, reason) do
34+
Repo.delete_all(
35+
from(p in PendingStatsDeletion, where: p.site_id in ^site_ids and p.reason == ^reason)
36+
)
37+
end
38+
39+
# Temporary. Bridges sites deleted before pending stats deletion tracking
40+
# existed. Finds sites with orphaned ClickHouse data (no matching Postgres
41+
# site) and records a pending deletion for each, so `ClickhouseCleanSites`
42+
# picks them up via `list/1`. Safe to run more than once. Remove
43+
# once it's been run in every environment.
44+
@spec backfill_orphaned_sites() :: {:ok, non_neg_integer()}
45+
def backfill_orphaned_sites() do
46+
already_tracked = MapSet.new(list())
47+
now = NaiveDateTime.utc_now(:second)
48+
49+
records =
50+
orphaned_clickhouse_site_ids()
51+
|> Enum.reject(&(&1 in already_tracked))
52+
|> Enum.map(fn site_id ->
53+
%{site_id: site_id, reason: :user_request, inserted_at: now, updated_at: now}
54+
end)
55+
56+
{count, _} = Repo.insert_all(PendingStatsDeletion, records)
57+
58+
{:ok, count}
59+
end
60+
61+
@all_stats_tables [
62+
"events_v2",
63+
"sessions_v2",
64+
"imported_browsers",
65+
"imported_devices",
66+
"imported_entry_pages",
67+
"imported_exit_pages",
68+
"imported_locations",
69+
"imported_operating_systems",
70+
"imported_pages",
71+
"imported_custom_events",
72+
"imported_sources",
73+
"imported_visitors"
74+
]
75+
76+
defp orphaned_clickhouse_site_ids() do
77+
pg_site_ids =
78+
from(s in Site.regular(), select: s.id)
79+
|> Repo.all()
80+
|> MapSet.new()
81+
82+
{:ok, ch} =
83+
Ch.start_link(ClickhouseRepo.get_config_without_ch_query_execution_timeout())
84+
85+
query =
86+
Enum.map_join(
87+
@all_stats_tables,
88+
"\nUNION DISTINCT\n",
89+
&"SELECT site_id FROM #{&1} GROUP BY site_id"
90+
)
91+
92+
%Ch.Result{columns: ["site_id"], rows: rows} =
93+
DBConnection.run(
94+
ch,
95+
fn conn -> Ch.query!(conn, query, [], timeout: :infinity) end,
96+
timeout: :infinity
97+
)
98+
99+
ch_site_ids = rows |> MapSet.new(fn [site_id] -> site_id end)
100+
101+
MapSet.difference(ch_site_ids, pg_site_ids) |> MapSet.to_list()
102+
end
103+
end

lib/plausible/purge.ex

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ defmodule Plausible.Purge do
4848
def delete_imported_stats!(%Plausible.Site{} = site) do
4949
Enum.each(Plausible.Imported.tables(), fn table ->
5050
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
51-
Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])
51+
Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id])
5252
end)
5353

5454
Plausible.Sites.clear_stats_start_date!(site)
@@ -71,7 +71,7 @@ defmodule Plausible.Purge do
7171
Enum.each(Plausible.Imported.tables(), fn table ->
7272
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"
7373

74-
Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id])
74+
Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id, import_id])
7575
end)
7676

7777
Plausible.Sites.clear_stats_start_date!(site)

lib/plausible/site/cache.ex

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ defmodule Plausible.Site.Cache do
6060

6161
@impl true
6262
def get_from_source(domain) do
63-
query = from s in base_db_query(), where: s.domain == ^domain
63+
query =
64+
from s in base_db_query(), where: s.domain == ^domain or s.domain_changed_from == ^domain
6465

6566
case Plausible.Repo.one(query) do
6667
{_, _, site = %Site{}} -> %Site{site | from_cache?: false}

lib/plausible/site/removal.ex

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,19 @@ defmodule Plausible.Site.Removal do
44
"""
55
use Plausible
66

7+
alias Plausible.PendingStatsDeletions
78
alias Plausible.Repo
89
alias Plausible.Teams
910

1011
import Ecto.Query
1112

12-
@spec run(Plausible.Site.t()) :: {:ok, map()}
13-
def run(site) do
13+
@spec run(Plausible.Site.t(), Keyword.t()) :: {:ok, map()}
14+
def run(site, opts \\ []) do
1415
Repo.transaction(fn ->
1516
site = Repo.preload(site, :team)
1617

18+
{:ok, pending_stats_deletion} = PendingStatsDeletions.store(site)
19+
1720
result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^site.domain))
1821

1922
Teams.Memberships.prune_guests(site.team)
@@ -27,7 +30,14 @@ defmodule Plausible.Site.Removal do
2730
Plausible.Billing.SiteLocker.update_for(site.team, send_email?: false)
2831
end
2932

30-
%{delete_all: result}
33+
cache_opts = Keyword.take(opts, [:cache_name, :multicall_timeout])
34+
:ok = Plausible.Site.Cache.broadcast_delete(site.domain, cache_opts)
35+
36+
if site.domain_changed_from do
37+
:ok = Plausible.Site.Cache.broadcast_delete(site.domain_changed_from, cache_opts)
38+
end
39+
40+
%{delete_all: result, pending_stats_deletion: pending_stats_deletion}
3141
end)
3242
end
3343
end

lib/plausible/sites.ex

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -398,13 +398,7 @@ defmodule Plausible.Sites do
398398
end
399399

400400
def ensure_stats_start_date(%Site{} = site) do
401-
start_date =
402-
[
403-
Plausible.Imported.earliest_import_start_date(site),
404-
native_stats_start_date(site)
405-
]
406-
|> Enum.reject(&is_nil/1)
407-
|> Enum.min(Date, fn -> nil end)
401+
start_date = compute_stats_start_date(site)
408402

409403
if start_date do
410404
site
@@ -421,6 +415,15 @@ defmodule Plausible.Sites do
421415
Plausible.Stats.Clickhouse.pageview_start_date_local(site)
422416
end
423417

418+
defp compute_stats_start_date(site) do
419+
[
420+
Plausible.Imported.earliest_import_start_date(site),
421+
native_stats_start_date(site)
422+
]
423+
|> Enum.reject(&is_nil/1)
424+
|> Enum.min(Date, fn -> nil end)
425+
end
426+
424427
def has_stats?(site) do
425428
!!ensure_stats_start_date(site).stats_start_date
426429
end

0 commit comments

Comments
 (0)