fix entity preview dupes

frontend/src/js/entity-history/actions.ts

@@ -165,21 +165,31 @@ export function useNewHistorySession() {
       return;
     }
 
-    const entityIds = result.csv
-      .slice(1)
-      .map((row) => {
-        for (const col of preferredIdColumns) {
-          // some values might be empty, search for defined values
-          if (row[col.columnIdx]) {
-            return {
-              id: row[col.columnIdx],
-              kind: col.idKind,
-            };
-          }
-          return null;
-        }
-      })
-      .filter(exists);
+    const distinctEntityIdsBarrier = new Set<string>();
+    const entityIds: EntityId[] = [];
+
+    for (const row of result.csv.slice(1)) {
+      // some id-cols might be empty, search for defined values
+      const col = preferredIdColumns.find((col) => exists(row[col.columnIdx]));
+
+      if (!col) {
+        continue;
+      }
+
+      const kind = col.idKind;
+      const id = row[col.columnIdx];
+
+      // Add any id just once, with how we select cols, this is likely to also ensure distinctness of entities, but not guaranteed
+      if (distinctEntityIdsBarrier.has(kind + id)) {
+        continue;
+      }
+
+      distinctEntityIdsBarrier.add(kind + id);
+      entityIds.push({
+        id: id,
+        kind: kind,
+      });
+    }
 
     if (entityIds.length === 0) {
       dispatch(loadHistoryData.failure(new Error("No entity IDs found")));
@@ -346,6 +356,7 @@ interface DateRow {
   from: Date;
   to: Date;
 }
+
 const transformEntityData = (
   data: { [key: string]: unknown }[],
   {

frontend/src/js/entity-history/saveAndLoad.ts

@@ -6,6 +6,7 @@ import { downloadBlob } from "../common/helpers/downloadBlob";
 import { toCSV } from "../file/csv";
 import { setMessage } from "../snack-message/actions";
 
+import type { SelectOptionT } from "../api/types";
 import { EntityIdsStatus } from "./History";
 import { LoadingPayload } from "./LoadHistoryDropzone";
 import { EntityId } from "./reducer";
@@ -54,9 +55,9 @@ export const useLoadHistory = ({
 
   return useCallback(
     ({ label, data }: { label: string; data: string[][] }) => {
-      const distinctEntityIds: Set<EntityId> = new Set();
+      const distinctEntityIds: Set<string> = new Set();
+      const loadedEntityIds: EntityId[] = [];
       const loadedEntityStatus: EntityIdsStatus = {};
-      const loadedEntityStatusOptionsRaw: string[] = [];
 
       // Expect data to be a CSV in format:
       // kind;id;status1;status2;...
@@ -68,23 +69,27 @@ export const useLoadHistory = ({
         const [kind, id] = row;
 
         // Deduplication is necessary for SecondaryId Queries
-        distinctEntityIds.add({ kind, id });
+        if (distinctEntityIds.has(kind + id)) {
+          continue;
+        }
+
+        distinctEntityIds.add(kind + id);
+        loadedEntityIds.push({ kind: kind, id: id });
 
         if (row.length > 2) {
           loadedEntityStatus[id] = row
             .slice(2)
             .filter((str) => str.length > 0)
             .map((s) => {
               const opt = s.trim();
-              loadedEntityStatusOptionsRaw.push(opt);
               return { label: opt, value: opt };
             });
         }
       }
 
-      const loadedEntityStatusOptions = [
-        ...new Set(loadedEntityStatusOptionsRaw),
-      ].map((item) => ({ label: item, value: item }));
+      const loadedEntityStatusOptions: SelectOptionT[] = [
+        ...new Set(Object.values(loadedEntityStatus).flatMap((val) => val)),
+      ];
 
       if (distinctEntityIds.size === 0) {
         dispatch(
@@ -96,8 +101,6 @@ export const useLoadHistory = ({
         return;
       }
 
-      const loadedEntityIds = [...distinctEntityIds];
-
       onLoadFromFile({
         label,
         loadedEntityIds,