fix

oms/scripts/migrate-clickhouse-events-partition.sh

@@ -299,11 +299,31 @@ build_reconcile_sql_for_month() {
   cat <<EOF
 INSERT INTO ${DB_NAME}.${DST_TABLE}
 SELECT src.*
-FROM ${DB_NAME}.${SRC_TABLE} AS src
-LEFT JOIN ${DB_NAME}.${DST_TABLE} AS dst ON src.log_id = dst.log_id
-WHERE src.time >= toDateTime('${month_start}')
-  AND src.time < addMonths(toDateTime('${month_start}'), 1)
-  AND empty(dst.log_id)
+FROM
+(
+  SELECT *
+  FROM ${DB_NAME}.${SRC_TABLE}
+  WHERE time >= toDateTime('${month_start}')
+    AND time < addMonths(toDateTime('${month_start}'), 1)
+) AS src
+LEFT ANTI JOIN
+(
+  SELECT log_id
+  FROM ${DB_NAME}.${DST_TABLE}
+  WHERE time >= toDateTime('${month_start}')
+    AND time < addMonths(toDateTime('${month_start}'), 1)
+) AS dst USING (log_id)
+EOF
+}
+
+month_diff_sql() {
+  local ym="$1"
+  cat <<EOF
+SELECT
+  '${ym}' AS ym,
+  (SELECT count() FROM ${DB_NAME}.${SRC_TABLE} WHERE toYYYYMM(time) = ${ym}) AS source_rows,
+  (SELECT count() FROM ${DB_NAME}.${DST_TABLE} WHERE toYYYYMM(time) = ${ym}) AS destination_rows,
+  source_rows - destination_rows AS missing_rows
 EOF
 }
 
@@ -339,6 +359,8 @@ if [[ $RECONCILE_ONLY -eq 1 ]]; then
     [[ -z "$ym" ]] && continue
     reconcile_sql="$(build_reconcile_sql_for_month "$ym")"
     run_sql_step "Reconcile month ${ym} missing rows" "$reconcile_sql"
+    printf '\nMonth diff after %s:\n' "$ym"
+    run_sql "$(month_diff_sql "$ym")"
   done <<< "$reconcile_months"
 
   old_rows="$(run_sql "SELECT count() FROM ${DB_NAME}.${SRC_TABLE} FORMAT TabSeparatedRaw")"