Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #390: Port release to 340

Browse Source 
Merge in RED/persistence-service from port-release-to-340 to master

* commit 'c17b7f75daf3bdbd5414ddb6e5e84e357171ee49':
  RED-3800 code cleanup
  Hotfix RED-3800 purge queue
  Hotfix RED-3800 reanslysis required status updates
  HOTFIX RED-3800 endpoint to reset text for all files that are not approved/deleted
  HOTFIX: RED-3800 ocr all files that have been ocrd
  HOTFIX: RED-3800 internal admin api
  migration manual run ctrl RED-3800
  NPE fix RED-3800
  RED-3800 Migration cleanup
  hotfix: migration to skip missing redaction logs (e.g. for deleted files w/o redaction log)
This commit is contained in:
Timo Bejan 2022-05-03 16:53:50 +02:00
commit 0b34daeaf3
23 changed files with 441 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java
View File

@ -19,10 +19,10 @@ public interface DossierTemplateStatsResource {
String DOSSIER_TEMPLATE_ID = "dossierTemplateId";
String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}";
@PostMapping(value = REST_PATH+DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
@PostMapping(value = REST_PATH + DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
List<DossierTemplateDictionaryStats> getDossierTemplateDictionaryStats(@RequestBody Set<String> dossierTemplateIds);
@PostMapping(value = REST_PATH+DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE )
@PostMapping(value = REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
DossierTemplateStats getDossierTemplateStats(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId);
@PostMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE)

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java
View File

@ -113,4 +113,7 @@ public class DossierEntity {
@Column(updatable = false, insertable = false, name = "dossier_status_id")
private String dossierStatusId;
public boolean isDeleted() {
return softDeletedTime != null || hardDeletedTime != null;
}
}

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/MigrationPersistenceService.java
View File

@ -29,20 +29,17 @@ public class MigrationPersistenceService {
var migrations = migrationRepository.findByVersionGreaterThan(lastProcessedVersion);
if (migrations == null || migrations.isEmpty()) {
if(version > lastProcessedVersion){
return false;
}
return true;
return version <= lastProcessedVersion;
}
return migrations.stream().filter(m -> m.getVersion() == version).findFirst().isPresent();
return migrations.stream().anyMatch(m -> m.getVersion() == version);
}
public Long getLatestProcessedVersion() {
var migrations = migrationRepository.findAll();
if (migrations == null || migrations.isEmpty()) {
if (migrations.isEmpty()) {
return null;
}
return migrations.stream().sorted(Comparator.comparing(MigrationEntity::getVersion).reversed()).collect(Collectors.toList()).get(0).getVersion();

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java
View File

@ -1,8 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
@ -20,7 +18,6 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
List<TypeEntity> findByDossierId(String dossierId);
@Query("select t from TypeEntity t where t.id = :typeId and t.softDeletedTime is null")
Optional<TypeEntity> findByIdAndNotDeleted(String typeId);
@Modifying

52
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java
View File

@ -1,25 +1,5 @@
package com.iqser.red.service.peristence.v1.server.controller;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.transaction.Transactional;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.peristence.v1.server.TextNormalizationUtilities;
import com.iqser.red.service.peristence.v1.server.service.StopwordService;
import com.iqser.red.service.peristence.v1.server.validation.DictionaryValidator;
@ -36,9 +16,25 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.transaction.Transactional;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
@RestController
@RequiredArgsConstructor
@ -80,8 +76,8 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
if (removeCurrent) {
entryPersistenceService.deleteAllEntriesForTypeId(typeId,currentVersion+1,dictionaryEntryType);
entryPersistenceService.addEntries(typeId,cleanEntries, currentVersion+1, dictionaryEntryType);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
} else {
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
}
@ -136,9 +132,9 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
}
@ -160,7 +156,7 @@ public class DictionaryController implements DictionaryResource {
checkForDuplicateLabels(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), typeRequest.getType(), typeRequest
.getLabel());
if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(),false)
if (dictionaryPersistenceService.getCumulatedTypes(typeRequest.getDossierTemplateId(), typeRequest.getDossierId(), false)
.stream()
.anyMatch(typeResult -> typeRequest.getDossierId() != null && typeResult.getDossierId() != null && typeRequest
.getDossierId()
@ -203,7 +199,7 @@ public class DictionaryController implements DictionaryResource {
@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
@RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) {
return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId,includeDeleted), Type.class);
return convert(dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, includeDeleted), Type.class);
}
@ -211,7 +207,7 @@ public class DictionaryController implements DictionaryResource {
public List<Type> getAllTypesForDossier(@PathVariable(DOSSIER_ID_PARAMETER_NAME) String dossierId,
@RequestParam(value = INCLUDE_DELETED_PARAMETER_NAME, required = false, defaultValue = "false") boolean includeDeleted) {
return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId,includeDeleted), Type.class);
return convert(dictionaryPersistenceService.getAllTypesForDossier(dossierId, includeDeleted), Type.class);
}

4
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -197,11 +197,11 @@ public class ReanalysisController implements ReanalysisResource {
if (force) {
filesToReanalyse.forEach(file -> {
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
});
} else {
filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> {
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1 ? true : false, true);
fileStatusService.setStatusReprocess(dossierId, file.getId(), filesToReanalyse.size() == 1, true);
});
}

151
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/internal/AdminInterfaceController.java Normal file
View File

@ -0,0 +1,151 @@
package com.iqser.red.service.peristence.v1.server.internal;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.stream.Collectors;
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/private/admin")
public class AdminInterfaceController {
private final FileManagementStorageService fileManagementStorageService;
private final FileStatusService fileStatusService;
private final FileManagementServiceSettings fileManagementServiceSettings;
private final DossierService dossierService;
private final AmqpAdmin amqpAdmin;
@PostMapping("/reset-file")
public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
}
@PostMapping("/reset-image-info")
public void resetImageInfo(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
if (fileManagementServiceSettings.isImageServiceEnabled()) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
fileStatusService.addToImageQueue(dossierId, fileId);
}
}
@PostMapping("/ocr")
public void forceOCR(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileStatusService.updateLastOCRTime(fileId);
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
}
@PostMapping("/ocr-all-files-that-have-already-been-ocrd")
public void forceOCR(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) {
if (dryRun) {
log.info("Dry run!");
}
var dossiers = dossierService.getAllDossiers();
for (var dossier : dossiers) {
if (!dossier.isDeleted()) {
var files = fileStatusService.getActiveFiles(dossier.getId());
var filesThatRequireOCR = files.stream()
.filter(f -> !f.isExcluded())
.filter(f -> !f.isExcludedFromAutomaticAnalysis())
.filter(f -> !f.isSoftOrHardDeleted())
.filter(f -> f.getLastOCRTime() != null)
.filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList());
for (var file : filesThatRequireOCR) {
log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}",
file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime());
if (!dryRun) {
fileStatusService.updateLastOCRTime(file.getId());
fileStatusService.setStatusOcrProcessing(file.getDossierId(), file.getId());
}
}
}
}
}
@PostMapping("/reset-text-for-all-files")
public void resetText(@RequestParam(value = "dryRun", defaultValue = "true") boolean dryRun) {
if (dryRun) {
log.info("Dry run!");
}
var dossiers = dossierService.getAllDossiers();
for (var dossier : dossiers) {
if (!dossier.isDeleted()) {
var files = fileStatusService.getActiveFiles(dossier.getId());
var filesThatRequireTextReset = files.stream()
.filter(f -> !f.isExcluded())
.filter(f -> !f.isExcludedFromAutomaticAnalysis())
.filter(f -> !f.isSoftOrHardDeleted())
.filter(f -> f.getWorkflowStatus() != WorkflowStatus.APPROVED).collect(Collectors.toList());
for (var file : filesThatRequireTextReset) {
log.info("Will OCR file: {} from dossier {} with status {} and processing status {} with last OCR time {}",
file.getId(), file.getDossierId(), file.getWorkflowStatus(), file.getProcessingStatus(), file.getLastOCRTime());
if (!dryRun) {
var dossierId = file.getDossierId();
var fileId = file.getId();
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
fileStatusService.setStatusFullReprocess(dossierId, fileId, true, true);
}
}
}
}
}
@PostMapping("/flush-queue")
public void resetText(@RequestParam() String queueName ) {
amqpAdmin.purgeQueue(queueName);
}
}

11
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/Migration.java
View File

@ -1,13 +1,11 @@
package com.iqser.red.service.peristence.v1.server.migration;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.MigrationPersistenceService;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@Data
@Slf4j
@ -21,11 +19,14 @@ public abstract class Migration {
@Setter
private MigrationPersistenceService migrationPersistenceService;
public void run() {
run(false);
}
public void run(boolean force) {
var latestProcessedVersion = migrationPersistenceService.getLatestProcessedVersion();
if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion)) {
if (!migrationPersistenceService.isProcessed(version, latestProcessedVersion) || force) {
log.info("Starting migration with name {} and version {}", name, version);
migrate();
migrationPersistenceService.insertMigration(name, version);

28
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationController.java Normal file
View File

@ -0,0 +1,28 @@
package com.iqser.red.service.peristence.v1.server.migration;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@Data
@RestController
@RequiredArgsConstructor
@RequestMapping("/private/migration")
public class MigrationController {
private final List<Migration> migrations;
@PostMapping
public void run(@RequestParam("migrationName") String migrationName, @RequestParam("force") boolean force) {
migrations.stream().filter(m -> m.getName().equalsIgnoreCase(migrationName) || m.getClass().getSimpleName().equalsIgnoreCase(migrationName))
.forEach(m -> m.run(force));
}
}

12
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DeleteRemovedManualAddRedactions7.java
View File

@ -1,20 +1,18 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.peristence.v1.server.service.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
@Slf4j
@Setter

25
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/DictionaryToEntityMigration2.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.ArrayList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
@ -13,11 +8,13 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@Slf4j
@Setter
@ -56,8 +53,9 @@ public class DictionaryToEntityMigration2 extends Migration {
files.forEach(file -> {
log.info("Start migration of file {}", file.getId());
if (file.getHardDeletedTime() == null) {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
try {
var newRedactionLogEntries = new ArrayList<RedactionLogEntry>();
var redactionLog = fileManagementStorageService.getRedactionLog(dossier.getId(), file.getId());
redactionLog.getRedactionLogEntry().forEach(entry -> {
@ -69,20 +67,21 @@ public class DictionaryToEntityMigration2 extends Migration {
if (entry.getType().startsWith("recommendation_")) {
entry.setType(entry.getType().substring(15));
entry.setRecommendation(true);
log.info("removed _recommendation");
log.info("removed _recommendation for file {} and annotation {}", file.getId(), entry.getId());
}
newRedactionLogEntries.add(entry);
});
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
log.info("Stored dossierId: {} and fileId: {}", dossier.getId(), file.getId());
if (newRedactionLogEntries.size() != redactionLog.getRedactionLogEntry().size()) {
redactionLog.setRedactionLogEntry(newRedactionLogEntries);
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.REDACTION_LOG, objectMapper.writeValueAsBytes(redactionLog));
log.info("Stored redactionLog for dossierId: {} and fileId: {}", dossier.getId(), file.getId());
}
} catch (JsonProcessingException e) {
throw new RuntimeException("Migration failed");
} catch (NotFoundException e) {
log.info("redactionLog {} does not exsist", file.getId());
return;
}
}
});

12
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/EntityTypesMigration4.java
View File

@ -1,20 +1,18 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Set;
@Slf4j
@Setter

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/IndexMigration1.java
View File

@ -1,13 +1,11 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.peristence.v1.server.service.IndexingService;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Slf4j
@Setter
@ -16,6 +14,8 @@ public class IndexMigration1 extends Migration {
private static final String NAME = "Recreate index and index all files";
private static final long VERSION = 1;
@Autowired
private IndexingService indexingService;
public IndexMigration1() {
@ -23,11 +23,6 @@ public class IndexMigration1 extends Migration {
super(NAME, VERSION);
}
@Autowired
private IndexingService indexingService;
@Override
protected void migrate() {

37
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java
View File

@ -32,7 +32,6 @@ public class MigrateHighlights3 extends Migration {
@Autowired
private FileManagementStorageService fileManagementStorageService;
public MigrateHighlights3() {
super(NAME, VERSION);
@ -48,24 +47,32 @@ public class MigrateHighlights3 extends Migration {
files.forEach(file -> {
if (file.getHardDeletedTime() == null) {
var untouchedExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
try {
if (fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN)) {
var untouchedExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
if (!untouchedExists) {
var originExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.ORIGIN);
if (!untouchedExists) {
var originExists = fileManagementStorageService.objectExists(dossier.getId(), file.getId(), FileType.ORIGIN);
if (!originExists) {
log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
return;
if (!originExists) {
log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
return;
}
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
}
var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
.fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
} else {
log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
}
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
} catch (Exception e) {
log.warn("Failed to extract text highlights for document: {}", file.getId());
}
var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
.fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
}
});
});

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/RemoveFalsePositiveManualRedactions6.java
View File

@ -1,18 +1,16 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import java.util.ArrayList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.controller.ManualRedactionController;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@Slf4j
@Setter
@ -59,6 +57,9 @@ public class RemoveFalsePositiveManualRedactions6 extends Migration {
}
});
}
log.info("Hard deleting false positive annotations for file: {} / {}", file.getId(), annotationIdsToRemove);
if (!annotationIdsToRemove.isEmpty()) {
annotationIdsToRemove.forEach(id -> addRedactionPersistenceService.hardDelete(file.getId(), id));
}

31
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/TypeToEntityMigration5.java
View File

@ -1,5 +1,17 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Locale;
@ -7,20 +19,6 @@ import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Setter
@Service
@ -65,6 +63,7 @@ public class TypeToEntityMigration5 extends Migration {
.filter(t -> t.getType().equals("false_positive"))
.findFirst();
if (falsePositive.isEmpty()) {
log.info("False positive type does no longer exist in dossierTemplate: {}. Skipping.", dossierTemplate.getName());
return;
}
typeIdsToDelete.add(falsePositive.get().getId());
@ -104,9 +103,7 @@ public class TypeToEntityMigration5 extends Migration {
log.info("Finished processing type {}", typeEntity.getType());
});
typeIdsToDelete.forEach(typeIdToDelete -> {
dictionaryController.deleteType(typeIdToDelete);
});
typeIdsToDelete.forEach(typeIdToDelete -> dictionaryController.deleteType(typeIdToDelete));
log.info("Finished false positive migration of dossierTemplate {}", dossierTemplate.getId());
});

23
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java
View File

@ -1,12 +1,5 @@
package com.iqser.red.service.peristence.v1.server.service;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -16,10 +9,15 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.SectionGrid;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
@Slf4j
@Service
@ -86,7 +84,7 @@ public class FileManagementStorageService {
}
public ImportedRedactions getImportedRedactions(String dossierId, String fileId){
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
InputStreamResource inputStreamResource;
try {
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
@ -101,11 +99,11 @@ public class FileManagementStorageService {
}
}
public boolean typeExists(String dossierId, String fileId, FileType type) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, type));
public boolean objectExists(String dossierId, String fileId, FileType origin) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, origin));
}
public void deleteObject(String dossierId, String fileId, FileType fileType) {
storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType));
@ -117,4 +115,5 @@ public class FileManagementStorageService {
storageService.deleteObject(storageId);
}
}

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -32,7 +32,7 @@ public class FileStatusProcessingUpdateService {
switch (analyzeResult.getMessageType()) {
case STRUCTURE_ANALYSE:
if (settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
if (settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
fileStatusService.addToNerQueue(dossierId, fileId);
} else {

92
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -1,26 +1,10 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.controller.RulesController;
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
@ -32,13 +16,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.Inter
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -46,9 +25,20 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.MessageType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@ -59,7 +49,6 @@ public class FileStatusService {
private final DossierPersistenceService dossierPersistenceService;
private final RabbitTemplate rabbitTemplate;
private final ObjectMapper objectMapper;
private final RulesController rulesController;
private final ManualRedactionProviderService manualRedactionProviderService;
private final FileManagementStorageService fileManagementStorageService;
private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
@ -129,7 +118,6 @@ public class FileStatusService {
}
public void setStatusProcessing(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.PROCESSING);
@ -199,9 +187,7 @@ public class FileStatusService {
return;
}
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.FULLREPROCESS);
if(requiresStructureAnalysis) {
if (requiresStructureAnalysis) {
log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
@ -215,24 +201,28 @@ public class FileStatusService {
protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
var fileStatus = fileStatusPersistenceService.getStatus(fileId);
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
if (fileStatus.isExcluded()) {
log.debug("File {} is excluded", fileStatus.getId());
if (fileEntity.isExcluded()) {
log.debug("File {} is excluded", fileEntity.getId());
return;
}
boolean reanalyse = isReanalyse(dossier, fileStatus);
var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper());
reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true);
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId,FileType.IMAGE_INFO)) {
boolean reanalyse = fileModel.isReanalysisRequired();
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
return;
}
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileModel.getProcessingStatus());
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
return;
@ -241,26 +231,26 @@ public class FileStatusService {
var analyseRequest = AnalyzeRequest.builder()
.messageType(messageType)
.dossierId(dossierId)
.analysisNumber(fileStatus.getNumberOfAnalyses() + 1)
.analysisNumber(fileModel.getNumberOfAnalyses() + 1)
.sectionsToReanalyse(sectionsToReanalyse)
.fileId(fileId)
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileStatus.getLastProcessed())
.fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileStatus.getExcludedPages())
.lastProcessed(fileModel.getLastProcessed())
.fileAttributes(convertAttributes(fileEntity.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileModel.getExcludedPages())
.build();
log.info("Add file: {} from dossier {} to Analysis queue with MessageType {}", fileId, dossierId, messageType);
if(messageType.equals(MessageType.REANALYSE)) {
if (messageType.equals(MessageType.REANALYSE)) {
setStatusProcessing(fileId);
} else {
setStatusFullProcessing(fileId);
}
try {
if(priority){
if (priority) {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_PRIORITY_QUEUE, objectMapper.writeValueAsString(analyseRequest));
} else {
rabbitTemplate.convertAndSend(MessagingConfiguration.REDACTION_QUEUE, objectMapper.writeValueAsString(analyseRequest));
@ -271,21 +261,10 @@ public class FileStatusService {
}
private boolean isReanalyse(DossierEntity dossier, FileEntity fileStatus) {
return !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId())
&& (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
.isAfter(fileStatus.getLastFileAttributeChange()));
}
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
ProcessingStatus processingStatus) {
if (!fileManagementStorageService.typeExists(dossierId, fileId, FileType.TEXT)) {
if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
return MessageType.STRUCTURE_ANALYSE;
}
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
@ -416,7 +395,7 @@ public class FileStatusService {
}
private void addToImageQueue(String dossierId, String fileId) {
public void addToImageQueue(String dossierId, String fileId) {
setStatusImageAnalyzing(fileId);
try {
@ -454,7 +433,6 @@ public class FileStatusService {
}
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights);

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -127,23 +127,26 @@ public class ManualRedactionService {
response.add(ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
if (actionPerformed) {
// in case of add to dict, there is no need to process surrounding text
reprocess(dossierId, fileId);
}else {
var manualTextRedactions = convert(response.stream()
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
}
var manualTextRedactions = convert(response.stream()
.map(r -> getAddRedaction(fileId, r.getAnnotationId()))
.filter(m -> !m.isAddToDictionary() && !m.isAddToDossierDictionary() && !m.isRectangle())
.collect(Collectors.toList()), ManualRedactionEntry.class, new ManualRedactionMapper());
if (!manualTextRedactions.isEmpty()) {
ManualRedactions manualRedactions = ManualRedactions.builder().entriesToAdd(new HashSet<>(manualTextRedactions)).build();
addManualRedactionToAnalysisQueue(dossierId, fileId, manualRedactions);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return response;
}

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java
View File

@ -33,16 +33,24 @@ public class ReanalysisRequiredStatusService {
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel)).iterator().next();
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), false).iterator().next();
}
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel, boolean ignoreProcessingStates) {
return enhanceFileStatusWithAnalysisRequirements(Collections.singletonList(fileModel), ignoreProcessingStates).iterator().next();
}
public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels) {
return enhanceFileStatusWithAnalysisRequirements(fileModels, false);
}
public List<FileModel> enhanceFileStatusWithAnalysisRequirements(List<FileModel> fileModels, boolean ignoreProcessingStates) {
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap = new HashMap<>();
Map<String, Long> dossierVersionMap = new HashMap<>();
Map<String, DossierEntity> dossierMap = new HashMap<>();
fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
});
@ -52,15 +60,18 @@ public class ReanalysisRequiredStatusService {
}
private AnalysisRequiredResult computeAnalysisRequired(FileModel fileStatus,
boolean ignoreProcessingStates,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, DossierEntity> dossierMap) {
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus())) {
if (ProcessingStatus.ERROR.equals(fileStatus.getProcessingStatus()) && !ignoreProcessingStates) {
return new AnalysisRequiredResult(false, true);
}
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus()) || ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())) {
if (ProcessingStatus.PROCESSED.equals(fileStatus.getProcessingStatus())
|| ProcessingStatus.UNPROCESSED.equals(fileStatus.getProcessingStatus())
|| ignoreProcessingStates) {
switch (fileStatus.getWorkflowStatus()) {
case NEW:

19
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java
View File

@ -9,11 +9,9 @@ import lombok.extern.slf4j.Slf4j;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.PostConstruct;
import java.util.List;
@Slf4j
@ -21,17 +19,9 @@ import java.util.List;
@RequiredArgsConstructor
public class AutomaticAnalysisJob implements Job {
@Value("${persistence-service.automaticAnalysis.pageFactor:500}")
private int pageFactor;
private final FileStatusService fileStatusService;
private final AmqpAdmin amqpAdmin;
private final FileManagementServiceSettings fileManagementServiceSettings;
@PostConstruct
protected void postConstruct() {
log.info("Automatic Analysis pageFactor: {}", pageFactor);
}
private final FileStatusService fileStatusService;
@Override
@ -55,12 +45,11 @@ public class AutomaticAnalysisJob implements Job {
var allStatusesIterator = allStatuses.iterator();
log.info("Files that require reanalysis: {}", allStatuses.size());
var worstCaseScenarioQueuedPages = 0;
var queuedFiles = 0;
while (worstCaseScenarioQueuedPages < pageFactor * consumerCount && allStatusesIterator.hasNext()) {
while (queuedFiles < (consumerCount * 5)) {
var next = allStatusesIterator.next();
// in case the file doesn't have numberOfPages set, we assume an average.
worstCaseScenarioQueuedPages += next.getNumberOfPages() <= 0 ? pageFactor : next.getNumberOfPages();
if (next.isFullAnalysisRequired()) {
log.info("Queued file: {} for automatic full analysis! ", next.getFilename());
@ -69,6 +58,8 @@ public class AutomaticAnalysisJob implements Job {
log.info("Queued file: {} for automatic reanalysis! ", next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false);
}
queuedFiles++;
}
}

94
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -1,38 +1,40 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.ManualChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import feign.FeignException;
import org.assertj.core.util.Lists;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.ManualChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import feign.FeignException;
import lombok.SneakyThrows;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
@ -57,6 +59,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
@Autowired
private FileManagementStorageService fileManagementStorageService;
@Autowired
private FileStatusPersistenceService fileStatusPersistenceService;
@Autowired
private ObjectMapper objectMapper;
@ -142,6 +147,49 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
}
@Test
@SneakyThrows
public void testAddToDictionaryRequiresReanalysis() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var type = typeProvider.testAndProvideType(dossierTemplate, null, "PII");
// assume file is already proccessed once, test that add to dict triggers reanalysis
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.TEXT, "dummy".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.NER_ENTITIES, "dummy".getBytes(StandardCharsets.UTF_8));
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.IMAGE_INFO, "dummy".getBytes(StandardCharsets.UTF_8));
fileStatusPersistenceService.updateProcessingStatus(file.getId(),ProcessingStatus.PROCESSED);
var addRedaction = manualRedactionClient.addAddRedaction(dossier.getId(), file.getId(), Collections.singletonList(AddRedactionRequest.builder()
.positions(List.of(Rectangle.builder().topLeftY(1).topLeftX(1).height(1).width(1).build()))
.section("section test")
.addToDictionary(true)
.addToDossierDictionary(false)
.status(AnnotationStatus.APPROVED)
.typeId(type.getId())
.user("user")
.reason("1")
.value("test")
.legalBasis("1")
.rectangle(true)
.textAfter("Text After")
.textBefore("Text Before")
.sourceId("SourceId")
.build())).iterator().next();
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getProcessingStatus()).isEqualTo(ProcessingStatus.PROCESSING);
}
@Test
@SneakyThrows
public void testManualRedaction() {
@ -295,7 +343,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.annotationStatus(AnnotationStatus.REQUESTED).build());
var loadedRemoveRedaction2 = manualRedactionClient.getRemoveRedaction(file.getId(), removeRedaction2.getAnnotationId());
assertThat(loadedRemoveRedaction2.getStatus()).isEqualTo(AnnotationStatus.REQUESTED);
assertThat(dictionaryClient.getDictionaryForType(type.getId(),null).getEntries().isEmpty());
assertThat(dictionaryClient.getDictionaryForType(type.getId(), null).getEntries().isEmpty());
assertThat(loadedRemoveRedaction2.isRemoveFromDictionary()).isEqualTo(true);
manualRedactionClient.updateRemoveRedactionStatus(dossier.getId(), file.getId(),
@ -448,7 +496,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertThat(manualRedactions.getComments()).isNotEmpty();
assertThat(manualRedactions.getResizeRedactions()).isNotEmpty();
List<String> annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
List<String> annotationIds = manualRedactions.getForceRedactions().stream().map(f -> f.getAnnotationId()).collect(Collectors.toList());
manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), annotationIds);
// manualRedactions.getForceRedactions()
// .forEach(e -> manualRedactionClient.deleteForceRedaction(dossier.getId(), file.getId(), List.of(e.getAnnotationId())));