|
| 1 | +defmodule Plausible.PendingStatsDeletions do |
| 2 | + @moduledoc """ |
| 3 | + Context for pending stats deletions |
| 4 | + """ |
| 5 | + |
| 6 | + import Ecto.Query |
| 7 | + |
| 8 | + alias Plausible.ClickhouseRepo |
| 9 | + alias Plausible.PendingStatsDeletion |
| 10 | + alias Plausible.Repo |
| 11 | + alias Plausible.Site |
| 12 | + |
| 13 | + @spec store(Site.t(), atom()) :: {:ok, PendingStatsDeletion.t()} |
| 14 | + def store(%Site{} = site, reason \\ :user_request) do |
| 15 | + Repo.insert(%PendingStatsDeletion{site_id: site.id, reason: reason}) |
| 16 | + end |
| 17 | + |
| 18 | + @spec list(atom()) :: [pos_integer()] |
| 19 | + def list(reason \\ :user_request) do |
| 20 | + from(p in PendingStatsDeletion, |
| 21 | + where: p.reason == ^reason, |
| 22 | + distinct: true, |
| 23 | + order_by: p.site_id, |
| 24 | + select: p.site_id |
| 25 | + ) |
| 26 | + |> Repo.all() |
| 27 | + end |
| 28 | + |
| 29 | + @spec clear([pos_integer()], atom()) :: {non_neg_integer(), nil} |
| 30 | + def clear(site_ids, reason \\ :user_request) |
| 31 | + def clear([], _reason), do: {0, nil} |
| 32 | + |
| 33 | + def clear(site_ids, reason) do |
| 34 | + Repo.delete_all( |
| 35 | + from(p in PendingStatsDeletion, where: p.site_id in ^site_ids and p.reason == ^reason) |
| 36 | + ) |
| 37 | + end |
| 38 | + |
| 39 | + # Temporary. Bridges sites deleted before pending stats deletion tracking |
| 40 | + # existed. Finds sites with orphaned ClickHouse data (no matching Postgres |
| 41 | + # site) and records a pending deletion for each, so `ClickhouseCleanSites` |
| 42 | + # picks them up via `list/1`. Safe to run more than once. Remove |
| 43 | + # once it's been run in every environment. |
| 44 | + @spec backfill_orphaned_sites() :: {:ok, non_neg_integer()} |
| 45 | + def backfill_orphaned_sites() do |
| 46 | + already_tracked = MapSet.new(list()) |
| 47 | + now = NaiveDateTime.utc_now(:second) |
| 48 | + |
| 49 | + records = |
| 50 | + orphaned_clickhouse_site_ids() |
| 51 | + |> Enum.reject(&(&1 in already_tracked)) |
| 52 | + |> Enum.map(fn site_id -> |
| 53 | + %{site_id: site_id, reason: :user_request, inserted_at: now, updated_at: now} |
| 54 | + end) |
| 55 | + |
| 56 | + {count, _} = Repo.insert_all(PendingStatsDeletion, records) |
| 57 | + |
| 58 | + {:ok, count} |
| 59 | + end |
| 60 | + |
| 61 | + @all_stats_tables [ |
| 62 | + "events_v2", |
| 63 | + "sessions_v2", |
| 64 | + "imported_browsers", |
| 65 | + "imported_devices", |
| 66 | + "imported_entry_pages", |
| 67 | + "imported_exit_pages", |
| 68 | + "imported_locations", |
| 69 | + "imported_operating_systems", |
| 70 | + "imported_pages", |
| 71 | + "imported_custom_events", |
| 72 | + "imported_sources", |
| 73 | + "imported_visitors" |
| 74 | + ] |
| 75 | + |
| 76 | + defp orphaned_clickhouse_site_ids() do |
| 77 | + pg_site_ids = |
| 78 | + from(s in Site.regular(), select: s.id) |
| 79 | + |> Repo.all() |
| 80 | + |> MapSet.new() |
| 81 | + |
| 82 | + {:ok, ch} = |
| 83 | + Ch.start_link(ClickhouseRepo.get_config_without_ch_query_execution_timeout()) |
| 84 | + |
| 85 | + query = |
| 86 | + Enum.map_join( |
| 87 | + @all_stats_tables, |
| 88 | + "\nUNION DISTINCT\n", |
| 89 | + &"SELECT site_id FROM #{&1} GROUP BY site_id" |
| 90 | + ) |
| 91 | + |
| 92 | + %Ch.Result{columns: ["site_id"], rows: rows} = |
| 93 | + DBConnection.run( |
| 94 | + ch, |
| 95 | + fn conn -> Ch.query!(conn, query, [], timeout: :infinity) end, |
| 96 | + timeout: :infinity |
| 97 | + ) |
| 98 | + |
| 99 | + ch_site_ids = rows |> MapSet.new(fn [site_id] -> site_id end) |
| 100 | + |
| 101 | + MapSet.difference(ch_site_ids, pg_site_ids) |> MapSet.to_list() |
| 102 | + end |
| 103 | +end |
0 commit comments