Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-7384' into 'master'

Browse Source 
RED-7384: fix db fields for manual changes and tenant details

Closes RED-7384

See merge request redactmanager/persistence-service!347
This commit is contained in:
Kilian Schüttler 2024-02-12 16:20:40 +01:00
commit 500d215069
7 changed files with 120 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -23,6 +23,8 @@ import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantSyncService;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantSyncEvent;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -50,14 +52,12 @@ public class SaasMigrationService implements TenantSyncService {
SaasMigrationStatusPersistenceService saasMigrationStatusPersistenceService;
DossierService dossierService;
ManualRedactionProviderService manualRedactionProviderService;
TenantProvider tenantProvider;
LayoutParsingRequestFactory layoutParsingRequestFactory;
RabbitTemplate rabbitTemplate;
FileManagementServiceSettings settings;
StorageService storageService;
SaasAnnotationIdMigrationService saasAnnotationIdMigrationService;
UncompressedFilesMigrationService uncompressedFilesMigrationService;
@ -80,9 +80,15 @@ public class SaasMigrationService implements TenantSyncService {
log.info("Finished uncompressed files migration ...");
int numberOfFiles = 0;
var dossiers = dossierService.getAllDossiers().stream().filter(dossier -> dossier.getHardDeletedTime() == null).toList();
var dossiers = dossierService.getAllDossiers()
.stream()
.filter(dossier -> dossier.getHardDeletedTime() == null)
.toList();
for (var dossier : dossiers) {
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId()).stream().filter(file -> file.getHardDeletedTime() == null).toList();
var files = fileStatusPersistenceService.getStatusesForDossier(dossier.getId())
.stream()
.filter(file -> file.getHardDeletedTime() == null)
.toList();
var migrationStati = saasMigrationStatusPersistenceService.findAll()
.stream()
.collect(Collectors.toMap(SaasMigrationStatusEntity::getFileId, SaasMigrationStatusEntity::getStatus));
@ -143,12 +149,12 @@ public class SaasMigrationService implements TenantSyncService {
try {
String dossierTemplateId = dossierService.getDossierById(dossierId).getDossierTemplateId();
rabbitTemplate.convertAndSend(MIGRATION_QUEUE,
MigrationRequest.builder()
.dossierTemplateId(dossierTemplateId)
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.build());
MigrationRequest.builder()
.dossierTemplateId(dossierTemplateId)
.dossierId(dossierId)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.build());
log.info("Layout Parsing finished for saas migration for tenant {} dossier {} and file {}", TenantContext.getTenantId(), dossierId, fileId);
} catch (Exception e) {
log.error("Queuing of entityLog migration failed with {}", e.getMessage());
@ -160,11 +166,14 @@ public class SaasMigrationService implements TenantSyncService {
private boolean layoutParsingFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE)) && storageService.objectExists(
TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT)) && storageService.objectExists(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES)) && storageService.objectExists(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE))
&& storageService.objectExists(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT))
&& storageService.objectExists(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId,
fileId,
FileType.DOCUMENT_PAGES))
&& storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION));
}
@ -232,10 +241,10 @@ public class SaasMigrationService implements TenantSyncService {
updateAnnotationIds(fileId, idMapping);
} catch (Exception e) {
String message = String.format("Error during annotation id migration for tenant %s dossier %s and file %s, cause %s",
TenantContext.getTenantId(),
dossierId,
fileId,
e.getMessage());
TenantContext.getTenantId(),
dossierId,
fileId,
e.getMessage());
saasMigrationStatusPersistenceService.updateErrorStatus(fileId, message);
log.error(message);
throw e;
@ -247,6 +256,7 @@ public class SaasMigrationService implements TenantSyncService {
if (saasMigrationStatusPersistenceService.countByStatus(SaasMigrationStatus.FINISHED) == saasMigrationStatusPersistenceService.countAll()) {
automaticAnalysisJob.startForTenant(TenantContext.getTenantId());
tenantProvider.updateDetails(TenantContext.getTenantId(), UpdateDetailsRequest.builder().key("persistence-service-ready").value(true).build());
log.info("Saas migration finished for tenantId {}, re-enabled scheduler", TenantContext.getTenantId());
}
}

52
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/AnalysisFlagsCalculationService.java
View File

@ -51,7 +51,8 @@ public class AnalysisFlagsCalculationService {
var viewedPagesForCurrentAssignee = viewedPagesPersistenceService.findViewedPages(fileId, file.getAssignee());
Map<Integer, OffsetDateTime> viewedPages = viewedPagesForCurrentAssignee.stream().collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime));
Map<Integer, OffsetDateTime> viewedPages = viewedPagesForCurrentAssignee.stream()
.collect(Collectors.toMap(ViewedPageEntity::getPage, ViewedPageEntity::getViewedTime));
boolean hasRedactions = false;
boolean hasHints = false;
@ -67,7 +68,8 @@ public class AnalysisFlagsCalculationService {
String type = getType(entry.getType());
var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges().get(entry.getChanges().size() - 1);
var lastChange = entry.getChanges().isEmpty() ? null : entry.getChanges()
.get(entry.getChanges().size() - 1);
var entryType = entry.getEntryType();
if (entry.getManualChanges() != null && !entry.getManualChanges().isEmpty()) {
@ -87,13 +89,14 @@ public class AnalysisFlagsCalculationService {
}
if (lastChange != null && //
(lastRedactionModification == null || lastChange.getDateTime().isAfter(lastRedactionModification)) && //
!entryType.equals(EntryType.HINT) && !entryType.equals(EntryType.RECOMMENDATION)) {
(lastRedactionModification == null || lastChange.getDateTime().isAfter(lastRedactionModification)) && //
!entryType.equals(EntryType.HINT) &&//
!entryType.equals(EntryType.RECOMMENDATION)) {
lastRedactionModification = lastChange.getDateTime();
}
if (!hasRedactions && entry.getState().equals(EntryState.APPLIED) && //
(entryType.equals(EntryType.ENTITY) || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.AREA))) {
(entryType.equals(EntryType.ENTITY) || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.AREA))) {
hasRedactions = true;
}
@ -105,23 +108,28 @@ public class AnalysisFlagsCalculationService {
hasImages = true;
}
OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions().get(0).getPageNumber());
OffsetDateTime viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions()
.get(0).getPageNumber());
if (file.getWorkflowStatus() != WorkflowStatus.APPROVED && lastChange != null && lastChange.getDateTime() != null && viewedPage != null && //
viewedPage.isBefore(lastChange.getDateTime())) {
viewedPage.isBefore(lastChange.getDateTime())) {
hasUpdates = true;
}
}
log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime);
if (file.isHasRedactions() == hasRedactions && file.isHasHints() == hasHints && file.isHasImages() == hasImages && file.isHasSuggestions() == hasSuggestions && file.isHasAnnotationComments() == hasComments && file.isHasUpdates() == hasUpdates) {
if (file.isHasRedactions() == hasRedactions
&& file.isHasHints() == hasHints
&& file.isHasImages() == hasImages
&& file.isHasSuggestions() == hasSuggestions
&& file.isHasAnnotationComments() == hasComments
&& file.isHasUpdates() == hasUpdates) {
log.info("Nothing Changed for file: {}", fileId);
} else {
fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);
}
if (lastRedactionModification != null && (file.getRedactionModificationDate() == null || file.getRedactionModificationDate().isBefore(lastRedactionModification))) {
fileStatusPersistenceService.setLastRedactionModificationDateForFile(fileId, lastRedactionModification);
}
@ -136,9 +144,7 @@ public class AnalysisFlagsCalculationService {
private void addIdsToTrace(String dossierId, String fileId) {
if (observationRegistry.getCurrentObservation() != null) {
observationRegistry.getCurrentObservation()
.highCardinalityKeyValue("dossierId", dossierId)
.highCardinalityKeyValue("fileId", fileId);
observationRegistry.getCurrentObservation().highCardinalityKeyValue("dossierId", dossierId).highCardinalityKeyValue("fileId", fileId);
}
}
@ -149,17 +155,17 @@ public class AnalysisFlagsCalculationService {
OffsetDateTime lastRedactionModification) {
return !entryType.equals(EntryType.HINT) && //
!entryType.equals(EntryType.RECOMMENDATION) && //
StringUtils.isNotEmpty(entry.getReason()) && //
(manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.RESIZE)) && //
manualChange.getProcessedDate() != null && //
(lastRedactionModification == null || manualChange.getProcessedDate().isAfter(lastRedactionModification));
!entryType.equals(EntryType.RECOMMENDATION) && //
StringUtils.isNotEmpty(entry.getReason()) && //
(manualChange.getManualRedactionType().equals(ManualRedactionType.ADD_LOCALLY) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.LEGAL_BASIS_CHANGE) || //
manualChange.getManualRedactionType().equals(ManualRedactionType.RESIZE)) && //
manualChange.getProcessedDate() != null && //
(lastRedactionModification == null || manualChange.getProcessedDate().isAfter(lastRedactionModification));
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadCompressingService.java
View File

@ -7,8 +7,11 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
import com.iqser.red.service.persistence.management.v1.processor.model.DownloadJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import lombok.RequiredArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class DownloadCompressingService {

33
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DownloadReadyJob.java
View File

@ -8,8 +8,10 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUti
import com.iqser.red.service.persistence.service.v1.api.shared.model.download.DownloadStatusValue;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.stereotype.Service;
@ -30,25 +32,28 @@ public class DownloadReadyJob implements Job {
log.debug("Running DownloadReadyJob");
tenantProvider.getTenants().forEach(tenant -> {
tenantProvider.getTenants()
.forEach(tenant -> {
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
return;
}
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
return;
}
TenantContext.setTenantId(tenant.getTenantId());
TenantContext.setTenantId(tenant.getTenantId());
downloadStatusPersistenceService.getStatus().stream().filter(d -> d.getStatus().equals(DownloadStatusValue.GENERATING)).forEach(download -> {
downloadStatusPersistenceService.getStatus()
.stream()
.filter(d -> d.getStatus().equals(DownloadStatusValue.GENERATING))
.forEach(download -> {
int numberOfFiles = download.getFiles().size();
var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId());
if (downloadRedactionFileStatus.size() == numberOfFiles) {
downloadCompressingService.markDownloadForCompression(download.getStorageId(),download.getUserId());
}
});
});
int numberOfFiles = download.getFiles().size();
var downloadRedactionFileStatus = downloadRedactionFileStatusRepository.findAllByDownloadStorageId(download.getStorageId());
if (downloadRedactionFileStatus.size() >= numberOfFiles) {
downloadCompressingService.markDownloadForCompression(download.getStorageId(), download.getUserId());
}
});
});
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/TenantUtils.java
View File

@ -4,9 +4,9 @@ import com.knecon.fforesight.tenantcommons.model.TenantResponse;
public class TenantUtils {
public static boolean isTenantReadyForPersistence(TenantResponse tenantResponse){
return tenantResponse.getDetails()!=null &&
tenantResponse.getDetails().getOrDefault("persistence-service-ready",false).equals(true);
public static boolean isTenantReadyForPersistence(TenantResponse tenantResponse) {
return tenantResponse.getDetails() != null && tenantResponse.getDetails().getOrDefault("persistence-service-ready", false).equals(true);
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -179,3 +179,5 @@ databaseChangeLog:
file: db/changelog/tenant/117-add-last-flag-calculation-date-to-file.yaml
- include:
file: db/changelog/tenant/118-delete-unapproved-manual-changes-and-drop-column-status.yaml
- include:
file: db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml

33
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/119-set-add-to-all-dossiers-correctly-in-manual-redaction-table.yaml Normal file
View File

@ -0,0 +1,33 @@
databaseChangeLog:
- changeSet:
id: setAddToAllDossiersFalse
author: your_username
changes:
- sql:
dbms: postgresql
endDelimiter: ';'
sql: |
UPDATE manual_redaction
SET add_to_all_dossiers = false
WHERE add_to_dictionary = false;
- sql:
dbms: postgresql
endDelimiter: ';'
sql: |
UPDATE manual_recategorization
SET add_to_all_dossiers = false
WHERE add_to_dictionary = false;
- sql:
dbms: postgresql
endDelimiter: ';'
sql: |
UPDATE manual_resize_redaction
SET add_to_all_dossiers = false
WHERE update_dictionary = false;
- sql:
dbms: postgresql
endDelimiter: ';'
sql: |
UPDATE id_removal
SET remove_from_all_dossiers = false
WHERE remove_from_dictionary = false;