feat: use new harvest API

.github/workflows/ci.yml

@@ -137,7 +137,7 @@ jobs:
         with:
           repository: opendatateam/udata
           path: ${{ env.UDATA_WORKING_DIR }}
-          ref: main
+          ref: switch_harvests_to_new_api_fields
 
       - name: Set up uv
         uses: astral-sh/setup-uv@v6

components/Harvesters/AdminHarvestersPage.vue

@@ -192,7 +192,7 @@ import HarvesterBadge from './HarvesterBadge.vue'
 import type { PaginatedArray } from '~/types/types'
 import AdminTable from '~/components/AdminTable/Table/AdminTable.vue'
 import AdminTableTh from '~/components/AdminTable/Table/AdminTableTh.vue'
-import type { HarvesterJob, HarvesterSource } from '~/types/harvesters'
+import type { HarvesterSource } from '~/types/harvesters'
 import { getHarvesterAdminUrl } from '~/utils/harvesters'
 
 const props = defineProps<{
@@ -201,7 +201,6 @@ const props = defineProps<{
 const { t } = useTranslation()
 const { formatDate } = useFormatDate()
 const config = useRuntimeConfig()
-const { $api } = useNuxtApp()
 
 const page = ref(1)
 const pageSize = ref(20)
@@ -225,38 +224,11 @@ const url = computed(() => {
 
 const { data: pageData, status } = await useAPI<PaginatedArray<HarvesterSource>>(url, { lazy: true })
 
-const jobs = ref<Record<string, HarvesterJob>>({})
-const jobsPromises = ref<Record<string, Promise<void>>>({})
-
-watchEffect(async () => {
-  if (!pageData.value) return
-
-  for (const source of pageData.value.data) {
-    if (!source.last_job) continue
-    if (source.last_job.id in jobsPromises.value) continue
-
-    jobsPromises.value[source.last_job.id] = $api<HarvesterJob>(`/api/1/harvest/job/${source.last_job.id}/`)
-      .then((job) => {
-        if (source.last_job) {
-          jobs.value[source.last_job.id] = job // Working because there is no conflicts between IDs from different types
-        }
-      })
-  }
-
-  await Promise.all(Object.values(jobsPromises.value))
-})
-
 function getHarvesterDataservices(harvester: HarvesterSource) {
-  if (!harvester.last_job || !jobs.value[harvester.last_job.id]) {
-    return 0
-  }
-  return jobs.value[harvester.last_job.id].items.filter(item => item.dataservice).length
+  return harvester.last_job?.items.by_type.dataservice ?? 0
 }
 
 function getHarvesterDatasets(harvester: HarvesterSource) {
-  if (!harvester.last_job || !jobs.value[harvester.last_job.id]) {
-    return 0
-  }
-  return jobs.value[harvester.last_job.id].items.filter(item => item.dataset).length
+  return harvester.last_job?.items.by_type.dataset ?? 0
 }
 </script>

components/Harvesters/JobBadge.vue

@@ -9,11 +9,11 @@
 
 <script setup lang="ts">
 import { throwOnNever } from '@datagouv/components-next'
-import type { HarvesterJob } from '~/types/harvesters'
+import type { HarvesterJob, HarvesterJobPreview } from '~/types/harvesters'
 import type { AdminBadgeType } from '~/types/types'
 
 const props = defineProps<{
-  job: HarvesterJob
+  job: HarvesterJob | HarvesterJobPreview
 }>()
 
 const { t } = useTranslation()

components/Harvesters/JobPage.vue

@@ -9,27 +9,30 @@
     <div class="text-sm text-mentionGrey space-y-1.5 mb-5">
       <div class="space-x-1">
         <RiCalendarEventLine class="inline size-3" />
-        <span>{{ $t('Débuté le :') }}</span>
+        <span>{{ $t('Débuté le :') }}</span>
         <span class="font-mono">{{ formatDate(job.started || job.created, { dateStyle: 'long', timeStyle: 'short' }) }}</span>
       </div>
       <div class="space-x-1">
         <RiCalendarEventLine class="inline size-3" />
-        <span>{{ $t('Terminé le :') }}</span>
+        <span>{{ $t('Terminé le :') }}</span>
         <span class="font-mono">{{ job.ended ? formatDate(job.ended, { dateStyle: 'long', timeStyle: 'short' }) : '—' }}</span>
       </div>
       <div class="space-x-1">
         <RiCheckboxCircleLine class="inline size-3" />
-        <span>{{ $t('Statut :') }}</span>
+        <span>{{ $t('Statut :') }}</span>
         <JobBadge :job />
       </div>
-      <div class="space-x-1">
+      <div
+        v-if="byStatus"
+        class="space-x-1"
+      >
         <RiInformationLine class="inline size-3" />
-        <span>{{ $t('Éléments :') }}</span>
+        <span>{{ $t('Éléments :') }}</span>
         <span class="space-x-2">
           <Tooltip class="inline">
             <span class="space-x-0.5 text-sm">
               <RiCheckLine class="inline size-3.5" />
-              <span>{{ job.items.filter((i) => i.status === 'done').length }}</span>
+              <span>{{ byStatus.done }}</span>
             </span>
             <template #tooltip>
               {{ $t('Éléments finis') }}
@@ -38,7 +41,7 @@
           <Tooltip class="inline">
             <span class="space-x-0.5 text-sm">
               <RiEyeOffLine class="inline size-3.5" />
-              <span>{{ job.items.filter((i) => i.status === 'skipped').length }}</span>
+              <span>{{ byStatus.skipped }}</span>
             </span>
             <template #tooltip>
               {{ $t('Éléments ignorés') }}
@@ -47,7 +50,7 @@
           <Tooltip class="inline">
             <span class="space-x-0.5 text-sm">
               <RiArchiveLine class="inline size-3.5" />
-              <span>{{ job.items.filter((i) => i.status === 'archived').length }}</span>
+              <span>{{ byStatus.archived }}</span>
             </span>
             <template #tooltip>
               {{ $t('Éléments archivés') }}
@@ -56,13 +59,13 @@
           <Tooltip class="inline">
             <span class="space-x-0.5 text-sm">
               <RiCloseLine class="inline size-3.5" />
-              <span>{{ job.items.filter((i) => i.status === 'failed').length }}</span>
+              <span>{{ byStatus.failed }}</span>
             </span>
             <template #tooltip>
               {{ $t('Éléments en échec') }}
             </template>
           </Tooltip>
-          <span>{{ $t('({count} au total)', { count: job.items.length }) }}</span>
+          <span>{{ $t('({count} au total)', { count: total }) }}</span>
         </span>
       </div>
     </div>
@@ -104,10 +107,10 @@
       <div class="flex flex-wrap gap-x-4 gap-y-2 items-center">
         <div class="w-full flex-none md:flex-1">
           <h2 class="inline text-sm font-bold uppercase mb-0">
-            {{ $t('{n} éléments | {n} élément | {n} éléments', job.items.length) }}
+            {{ $t('{n} éléments | {n} élément | {n} éléments', total) }}
           </h2>
           <span
-            v-if="preview && job.items.length >= config.public.harvesterPreviewMaxItems"
+            v-if="preview && total >= config.public.harvesterPreviewMaxItems"
             class="ml-3 text-gray-medium"
           >{{ $t('Seuls les {n} premiers éléments sont affichés dans la prévisualisation.', config.public.harvesterPreviewMaxItems) }}</span>
         </div>
@@ -119,7 +122,7 @@
             v-model="selectedItemStatus"
             :placeholder="$t('Filtrer par statut')"
             :label="$t('Filtrer par statut')"
-            :options="itemStatus"
+            :options="itemStatusOptions"
             :display-value="(option: { label: string }) => option.label"
             :multiple="false"
             class="mb-0"
@@ -128,7 +131,7 @@
         </div>
       </div>
       <AdminTable
-        v-if="job.items.length"
+        v-if="total"
         class="fr-mb-2w"
       >
         <thead>
@@ -158,7 +161,7 @@
         </thead>
         <tbody>
           <tr
-            v-for="item in paginatedItems"
+            v-for="item in displayedItems"
             :key="item.remote_id"
           >
             <td>
@@ -221,9 +224,10 @@
         </tbody>
       </AdminTable>
       <Pagination
+        v-if="!preview"
         :page="page"
         :page-size="pageSize"
-        :total-results="currentItems.length"
+        :total-results="displayedTotal"
         @change="(changedPage: number) => page = changedPage"
       />
     </div>
@@ -281,39 +285,21 @@ import { RiAlertLine, RiArchiveLine, RiCalendarEventLine, RiCheckboxCircleLine,
 import AdminTable from '~/components/AdminTable/Table/AdminTable.vue'
 import AdminTableTh from '~/components/AdminTable/Table/AdminTableTh.vue'
 import JobBadge from '~/components/Harvesters/JobBadge.vue'
-import type { HarvesterJob, HarvestItem } from '~/types/harvesters'
+import type { HarvesterJob, HarvesterJobPreview, HarvestItem, HarvestItemStatus } from '~/types/harvesters'
 import type { AdminBadgeType } from '~/types/types'
 
 const config = useRuntimeConfig()
 const { t } = useTranslation()
 const { formatDate } = useFormatDate()
 
 const props = withDefaults(defineProps<{
-  job: HarvesterJob
-  preview?: boolean
+  job: HarvesterJob | HarvesterJobPreview
+  items?: Array<HarvestItem>
 }>(), {
-  preview: false,
-})
-
-const page = ref(1)
-const pageSize = ref(15)
-const currentItems = ref<Array<HarvestItem>>(props.job.items)
-
-const selectedItemStatus = ref<{ id: string, label: string, type: AdminBadgeType } | null>(null)
-
-watch(selectedItemStatus, () => {
-  page.value = 1
-  currentItems.value = props.job.items
-  const status = selectedItemStatus.value
-  if (status)
-    currentItems.value = currentItems.value.filter(item => item.status == status.id)
+  items: () => [],
 })
 
-const paginatedItems = computed(() => {
-  return currentItems.value.slice((page.value - 1) * pageSize.value, page.value * pageSize.value)
-})
-
-const itemStatusMap: Record<HarvestItem['status'], { label: string, type: AdminBadgeType }> = {
+const itemStatusMap: Record<HarvestItemStatus, { label: string, type: AdminBadgeType }> = {
   pending: { label: t('En attente'), type: 'secondary' },
   started: { label: t('Commencé'), type: 'primary' },
   done: { label: t('Terminé'), type: 'success' },
@@ -322,7 +308,23 @@ const itemStatusMap: Record<HarvestItem['status'], { label: string, type: AdminB
   archived: { label: t('Archivé'), type: 'secondary' },
 }
 
-const itemStatus = Object.entries(itemStatusMap).map(([id, status]) => ({ id, ...status }))
+const itemStatusOptions = Object.entries(itemStatusMap).map(([id, status]) => ({ id: id as HarvestItemStatus, ...status }))
+
+const page = ref(1)
+const pageSize = ref(15)
+const selectedItemStatus = ref<{ id: HarvestItemStatus, label: string, type: AdminBadgeType } | null>(null)
+
+watch(selectedItemStatus, () => {
+  page.value = 1
+})
+
+const { preview, displayedItems, displayedTotal, total, byStatus } = await useJobItems(
+  props.job,
+  () => props.items,
+  page,
+  pageSize,
+  selectedItemStatus,
+)
 
 function getStatus(item: HarvestItem): { label: string, type: AdminBadgeType } {
   return itemStatusMap[item.status]

components/Harvesters/PreviewStep.vue

@@ -8,7 +8,7 @@
   <div v-else-if="job">
     <JobPage
       :job
-      preview
+      :items="job.items"
     />
     <div class="flex items-center justify-between">
       <BrandedButton
@@ -31,7 +31,7 @@
 import { BrandedButton } from '@datagouv/components-next'
 import JobPage from './JobPage.vue'
 import PreviewLoader from './PreviewLoader.vue'
-import type { HarvesterForm, HarvesterJob } from '~/types/harvesters'
+import type { HarvesterForm, HarvesterJobPreview } from '~/types/harvesters'
 
 const props = defineProps<{
   harvesterForm: HarvesterForm
@@ -45,13 +45,13 @@ defineEmits<{
 const { $api } = useNuxtApp()
 
 const loading = ref(false)
-const job = ref<HarvesterJob | null>(null)
+const job = ref<HarvesterJobPreview | null>(null)
 
 onMounted(async () => {
   loading.value = true
 
   try {
-    job.value = await $api<HarvesterJob>('/api/1/harvest/source/preview', {
+    job.value = await $api<HarvesterJobPreview>('/api/1/harvest/source/preview', {
       method: 'POST',
       body: harvesterToApi(props.harvesterForm),
     })

composables/useJobItems.ts

@@ -0,0 +1,49 @@
+import type { MaybeRefOrGetter, Ref } from 'vue'
+import type { HarvesterJob, HarvesterJobPreview, HarvestItem, HarvestItemStatus } from '~/types/harvesters'
+import type { PaginatedArray } from '~/types/types'
+
+// Real jobs expose items as a paginated subresource (server-side filter &
+// pagination) and carry by_status / total counters on the link object.
+// Previews are transient — not persisted — so items travel inline as an array,
+// no fetch happens, and per-status counters would be misleading on such a
+// small sample (capped server-side), hence `byStatus` stays null.
+//
+// This composable hides that divergence so JobPage consumes plain
+// ComputedRef values and stays a pure view component.
+function isPreviewJob(job: HarvesterJob | HarvesterJobPreview): job is HarvesterJobPreview {
+  return Array.isArray(job.items)
+}
+
+export async function useJobItems(
+  job: HarvesterJob | HarvesterJobPreview,
+  inlineItems: MaybeRefOrGetter<Array<HarvestItem>>,
+  page: Ref<number>,
+  pageSize: Ref<number>,
+  selectedStatus: Ref<{ id: HarvestItemStatus } | null>,
+) {
+  if (isPreviewJob(job)) {
+    return {
+      preview: true as const,
+      displayedItems: computed(() => toValue(inlineItems)),
+      displayedTotal: computed(() => toValue(inlineItems).length),
+      total: computed(() => toValue(inlineItems).length),
+      byStatus: null,
+    }
+  }
+
+  const itemsUrl = computed(() => `/api/1/harvest/job/${job.id}/items/`)
+  const itemsParams = computed(() => ({
+    page: page.value,
+    page_size: pageSize.value,
+    ...(selectedStatus.value ? { status: selectedStatus.value.id } : {}),
+  }))
+  const { data: itemsPage } = await useAPI<PaginatedArray<HarvestItem>>(itemsUrl, { lazy: true, query: itemsParams })
+
+  return {
+    preview: false as const,
+    displayedItems: computed(() => itemsPage.value?.data ?? []),
+    displayedTotal: computed(() => itemsPage.value?.total ?? 0),
+    total: computed(() => job.items.total),
+    byStatus: computed(() => job.items.by_status),
+  }
+}

pages/admin/harvesters/[id]/configuration.vue

@@ -35,7 +35,7 @@
           >
             <JobPage
               :job="previewJob"
-              preview
+              :items="previewJob.items"
             />
           </div>
         </ModalWithButton>
@@ -97,7 +97,7 @@ import { BannerAction, BrandedButton, toast } from '@datagouv/components-next'
 import DescribeHarvester from '~/components/Harvesters/DescribeHarvester.vue'
 import JobPage from '~/components/Harvesters/JobPage.vue'
 import PreviewLoader from '~/components/Harvesters/PreviewLoader.vue'
-import type { HarvesterForm, HarvesterJob, HarvesterSource } from '~/types/harvesters'
+import type { HarvesterForm, HarvesterJobPreview, HarvesterSource } from '~/types/harvesters'
 
 const route = useRoute()
 const { $api } = useNuxtApp()
@@ -151,11 +151,11 @@ const save = async () => {
   }
 }
 
-const previewJob = ref<HarvesterJob | null>(null)
+const previewJob = ref<HarvesterJobPreview | null>(null)
 const preview = async () => {
   if (!harvesterForm.value) throw new Error('No harvester form')
 
-  previewJob.value = await $api<HarvesterJob>('/api/1/harvest/source/preview', {
+  previewJob.value = await $api<HarvesterJobPreview>('/api/1/harvest/source/preview', {
     method: 'POST',
     body: harvesterToApi(harvesterForm.value),
   })