Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-10264: add migration fixing missing legalbasis

Browse Source
This commit is contained in:
Kilian Schüttler 2024-10-30 14:20:20 +01:00
parent b174ca57b1
commit dddc608c34
31 changed files with 282 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -21,6 +21,7 @@ import org.springframework.web.multipart.MultipartFile;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DatasetExchangeService;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeImportService;
import com.iqser.red.service.persistence.management.v1.processor.migration.MigrationController;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper;
import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService;

59
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DossierTemplateResource.java vendored
View File

@ -95,35 +95,36 @@ public interface DossierTemplateResource {
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@PostMapping(value = DOSSIER_TEMPLATE_PATH + IMPORT_PATH, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Receives an archive to import", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400", description = "Import process stuck in one of the steps:"
+ "0 - Reading the archive content\n"
+ "\n"
+ "1 - store information about the dossier template\n"
+ "\n"
+ "2 - store the colors\n"
+ "\n"
+ "3 - store the watermarks\n"
+ "\n"
+ "4 - store the dossier status\n"
+ "\n"
+ "5 - store the dossier attributes\n"
+ "\n"
+ "6 - store the file attributes\n"
+ "\n"
+ "7 - store the report templates\n"
+ "\n"
+ "8 - store the legal basis\n"
+ "\n"
+ "9 - store the file attribute configuration\n"
+ "\n"
+ "10 - store the component definitions\n"
+ "\n"
+ "11 - store the component mappings\n"
+ "\n"
+ "12 - store the types and entities\n"
+ "\n"
+ "13 - store the rules and component rules"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
@Operation(summary = "Receives an archive to import", description = """
Import process stuck in one of the steps:"
0 - Reading the archive content
1 - store information about the dossier template
2 - store the colors
3 - store the watermarks
4 - store the dossier status
5 - store the dossier attributes
6 - store the file attributes
7 - store the report templates
8 - store the legal basis
9 - store the file attribute configuration
10 - store the component definitions
11 - store the component mappings
12 - store the types and entities
13 - store the rules and component rules""")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "Archive have successfully imported"), @ApiResponse(responseCode = "400"), @ApiResponse(responseCode = "404", description = "The dossier template to update does not exist")})
DossierTemplateModel importDossierTemplate(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file,
@RequestParam(value = DOSSIER_TEMPLATE_ID, required = false) String dossierTemplateId,
@RequestParam(value = "updateExistingDossierTemplate", required = false, defaultValue = "false") boolean updateExistingDossierTemplate);

4
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -75,8 +75,8 @@ dependencies {
api("com.opencsv:opencsv:5.9")
implementation("com.google.protobuf:protobuf-java:4.27.1")
implementation("org.mapstruct:mapstruct:1.5.5.Final")
annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
implementation("org.mapstruct:mapstruct:1.6.2")
annotationProcessor("org.mapstruct:mapstruct-processor:1.6.2")
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
testImplementation("org.testcontainers:postgresql:1.17.1")

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualAddExportModel.java
View File

@ -11,13 +11,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualForceExportModel.java
View File

@ -10,7 +10,6 @@ import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualLegalBasisChangeExportModel.java
View File

@ -8,7 +8,6 @@ import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ManualLegalBasisChangeExportModel {

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRecategorizationExportModel.java
View File

@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualRemoveExportModel.java
View File

@ -10,7 +10,6 @@ import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ManualRemoveExportModel {

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/models/manualchanges/ManualResizeExportModel.java
View File

@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/AnnotationEntityId.java
View File

@ -12,7 +12,6 @@ import lombok.NoArgsConstructor;
@Embeddable
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class AnnotationEntityId implements Serializable {

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java
View File

@ -17,14 +17,12 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Entity
@Table(name = "id_removal")
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class IdRemovalEntity implements IBaseAnnotation {

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java
View File

@ -15,7 +15,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
//@Builder // disabled, cause the ManualChangesExportMapper will not map addToDictionary and addToAllDossiers at all
@AllArgsConstructor
@NoArgsConstructor
@Entity

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java
View File

@ -10,12 +10,10 @@ import jakarta.persistence.Entity;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Entity

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRecategorizationEntity.java
View File

@ -17,12 +17,10 @@ import jakarta.persistence.Entity;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Entity

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java
View File

@ -22,13 +22,11 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Entity
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Table(name = "manual_redaction")

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java
View File

@ -19,12 +19,10 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@Entity

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/projection/DossierStatsFileProjection.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.projection;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Set;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import jakarta.persistence.Convert;
public interface DossierStatsFileProjection {

63
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ManualChangesExportMapper.java
View File

@ -73,65 +73,76 @@ public interface ManualChangesExportMapper {
@AfterMapping
default void setFileStatusAndId(ManualLegalBasisChangeExportModel model,
@MappingTarget ManualLegalBasisChangeEntity.ManualLegalBasisChangeEntityBuilder entity,
@Context FileEntity file, @Context String userId) {
@MappingTarget ManualLegalBasisChangeEntity entity,
@Context FileEntity file,
@Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
@AfterMapping
default void setFileStatusAndId(ManualRecategorizationExportModel model,
@MappingTarget ManualRecategorizationEntity.ManualRecategorizationEntityBuilder entity,
@Context FileEntity file, @Context String userId) {
@MappingTarget ManualRecategorizationEntity entity,
@Context FileEntity file,
@Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
@AfterMapping
default void setFileStatusAndId(ManualAddExportModel model, @MappingTarget ManualRedactionEntryEntity.ManualRedactionEntryEntityBuilder entity, @Context FileEntity file, @Context String userId) {
default void setFileStatusAndId(ManualAddExportModel model,
@MappingTarget ManualRedactionEntryEntity entity,
@Context FileEntity file,
@Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
@AfterMapping
default void setFileStatusAndId(ManualResizeExportModel model, @MappingTarget ManualResizeRedactionEntity.ManualResizeRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
default void setFileStatusAndId(ManualResizeExportModel model,
@MappingTarget ManualResizeRedactionEntity entity,
@Context FileEntity file,
@Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
@AfterMapping
default void setFileStatusAndId(ManualForceExportModel model, @MappingTarget ManualForceRedactionEntity.ManualForceRedactionEntityBuilder entity, @Context FileEntity file, @Context String userId) {
default void setFileStatusAndId(ManualForceExportModel model,
@MappingTarget ManualForceRedactionEntity entity,
@Context FileEntity file,
@Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
@AfterMapping
default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity.IdRemovalEntityBuilder entity, @Context FileEntity file, @Context String userId) {
default void setFileStatusAndId(ManualRemoveExportModel model, @MappingTarget IdRemovalEntity entity, @Context FileEntity file, @Context String userId) {
AnnotationEntityId annotationEntityId = new AnnotationEntityId(model.getAnnotationId(), file.getId());
entity.id(annotationEntityId);
entity.fileStatus(file);
entity.user(userId);
entity.setId(annotationEntityId);
entity.setFileStatus(file);
entity.setUser(userId);
}
}

44
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationManualChangesUpdateService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.migration;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import org.springframework.stereotype.Service;
@ -12,8 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.annotati
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import lombok.RequiredArgsConstructor;
@ -39,26 +38,27 @@ public class SaasMigrationManualChangesUpdateService {
if (unprocessedManualAdd.isAddToDictionary() || unprocessedManualAdd.isAddToAllDossiers()) {
// copy pending dict change to a new one with a different id. Can't reuse the same one, as it's the primary key of the table.
// It has no functionality, its only there, such that the UI can show a pending change.
ManualRedactionEntryEntity pendingDictAdd = ManualRedactionEntryEntity.builder()
.id(buildSecondaryId(unprocessedManualAdd.getId(), fileId))
.user(unprocessedManualAdd.getUser())
.typeId(unprocessedManualAdd.getTypeId())
.value(unprocessedManualAdd.getValue())
.reason(unprocessedManualAdd.getReason())
.legalBasis(unprocessedManualAdd.getLegalBasis())
.section(unprocessedManualAdd.getSection())
.rectangle(unprocessedManualAdd.isRectangle())
.addToDictionary(unprocessedManualAdd.isAddToDictionary())
.addToAllDossiers(unprocessedManualAdd.isAddToAllDossiers())
.dictionaryEntryType(DictionaryEntryType.ENTRY)
.requestDate(unprocessedManualAdd.getRequestDate())
.positions(new ArrayList<>(unprocessedManualAdd.getPositions())) // copy to new List
.fileStatus(unprocessedManualAdd.getFileStatus())
.textBefore(unprocessedManualAdd.getTextBefore())
.textAfter(unprocessedManualAdd.getTextAfter())
.sourceId(unprocessedManualAdd.getSourceId())
.typeIdsOfModifiedDictionaries(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries())
.build();
ManualRedactionEntryEntity pendingDictAdd = new ManualRedactionEntryEntity(buildSecondaryId(unprocessedManualAdd.getId(), fileId),
unprocessedManualAdd.getUser(),
unprocessedManualAdd.getTypeId(),
unprocessedManualAdd.getValue(),
unprocessedManualAdd.getReason(),
unprocessedManualAdd.getLegalBasis(),
unprocessedManualAdd.getSection(),
unprocessedManualAdd.isRectangle(),
unprocessedManualAdd.isAddToDictionary(),
unprocessedManualAdd.isAddToAllDossiers(),
unprocessedManualAdd.isAddToDossierDictionary(),
DictionaryEntryType.ENTRY,
unprocessedManualAdd.getRequestDate(),
null,
null,
new ArrayList<>(unprocessedManualAdd.getPositions()),
unprocessedManualAdd.getFileStatus(),
unprocessedManualAdd.getTextBefore(),
unprocessedManualAdd.getTextAfter(),
unprocessedManualAdd.getSourceId(),
new HashSet<>(unprocessedManualAdd.getTypeIdsOfModifiedDictionaries()));
addRedactionPersistenceService.update(pendingDictAdd);

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -380,7 +380,7 @@ public class SaasMigrationService implements TenantSyncService {
private AnnotationEntityId buildAnnotationId(String fileId, String annotationId) {
return AnnotationEntityId.builder().fileId(fileId).annotationId(annotationId).build();
return new AnnotationEntityId(annotationId, fileId);
}

90
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualChangesReorderingMigration28.java Normal file
View File

@ -0,0 +1,90 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.time.OffsetDateTime;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.migration.Migration;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class ManualChangesReorderingMigration28 extends Migration {
private static final String NAME = "Migration for reordering mixed up manual changes";
private static final long VERSION = 28;
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final FileStatusService fileStatusService;
public ManualChangesReorderingMigration28(EntityLogEntryDocumentRepository entityLogEntryDocumentRepository,
AddRedactionPersistenceService addRedactionPersistenceService,
FileStatusService fileStatusService) {
super(NAME, VERSION);
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
this.addRedactionPersistenceService = addRedactionPersistenceService;
this.fileStatusService = fileStatusService;
}
@Override
protected void migrate() {
log.info("Searching for entityLogEntries of type ENTITY with state APPLIED and no legalbasis");
List<EntryWithManualChangesProjection> entriesWithManualChanges = entityLogEntryDocumentRepository.findAppliedEntitiesWhereLegalBasisEmpty();
log.info("Found {} applied entries with no legal basis", entriesWithManualChanges.size());
int numberOfChanged = 0;
Set<FileAndDossier> affectedFiles = new HashSet<>();
for (EntryWithManualChangesProjection entry : entriesWithManualChanges) {
if (entry.getManualChanges().isEmpty()) {
continue;
}
Optional<ManualChange> optionalAdd = entry.getManualChanges()
.stream()
.filter(mc -> mc.getManualRedactionType().equals(ManualRedactionType.ADD))
.findFirst();
if (optionalAdd.isEmpty()) {
continue;
}
ManualChange add = optionalAdd.get();
ManualChange first = entry.getManualChanges()
.get(0);
if (add.equals(first)) {
continue;
}
OffsetDateTime firstTimestamp = first.getRequestedDate().minusSeconds(1);
String dossierId = entry.getEntityLogId().split("/")[0];
String fileId = entry.getEntityLogId().split("/")[1];
int i = addRedactionPersistenceService.updateRequestDate(fileId, entry.getEntryId(), firstTimestamp);
if (i != 0) {
affectedFiles.add(new FileAndDossier(fileId, dossierId));
numberOfChanged += i;
}
}
log.info("Updated request date of {} manual add redactions in {} files.", numberOfChanged, affectedFiles.size());
log.info("Reanalyzing affected files");
for (FileAndDossier affectedFile : affectedFiles) {
fileStatusService.setStatusReprocess(affectedFile.dossierId(), affectedFile.fileId(), false);
}
}
private record FileAndDossier(String fileId, String dossierId) {
}
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/MigrateImportedRedactionsFiles17.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.migration.migrations;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -14,7 +13,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;

8
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java
View File

@ -8,7 +8,7 @@ import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
@ -36,20 +36,20 @@ public class DossierIdFileIdRequestValidator {
}
if (!fileIds.isEmpty()) {
Set<FileStatus> availableFiles = fileStatusManagementService.findAllByIds(fileIds)
Set<FileModel> availableFiles = fileStatusManagementService.findAllByIds(fileIds)
.stream()
.filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
.collect(Collectors.toSet());
Set<String> mentionedDossierIds = availableFiles.stream()
.map(FileStatus::getDossierId)
.map(FileModel::getDossierId)
.collect(Collectors.toSet());
Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, mentionedDossierIds));
Set<String> availableFileIds = availableFiles.stream()
.filter(file -> availableDossierIds.contains(file.getDossierId()))
.map(FileStatus::getFileId)
.map(FileModel::getId)
.collect(Collectors.toSet());
Set<String> nonAvailableFiles = Sets.difference(fileIds, availableFileIds);

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java
View File

@ -10,7 +10,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -202,7 +201,7 @@ public class FileStatusManagementService {
}
public List<FileStatus> findAllByIds(Set<String> fileIds) {
public List<FileModel> findAllByIds(Set<String> fileIds) {
return fileStatusService.findAllByIds(fileIds);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -11,8 +11,6 @@ import java.util.Set;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
@ -25,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -62,7 +61,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.NerServiceRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
@ -1092,9 +1090,12 @@ public class FileStatusService {
}
public List<FileStatus> findAllByIds(Set<String> fileIds) {
public List<FileModel> findAllByIds(Set<String> fileIds) {
return fileStatusPersistenceService.findAllByIds(fileIds).stream().map(entity -> MagicConverter.convert(entity, FileStatus.class)).collect(Collectors.toList());
return fileStatusPersistenceService.findAllByIds(fileIds)
.stream()
.map(entity -> MagicConverter.convert(entity, FileModel.class, new FileModelMapper()))
.collect(Collectors.toList());
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java
View File

@ -159,4 +159,11 @@ public class AddRedactionPersistenceService {
manualRedactionRepository.undeleteByFileId(fileId, deletionTime);
}
@Transactional
public int updateRequestDate(String fileId, String annotationId, OffsetDateTime requestDate) {
return manualRedactionRepository.updateRequestDateById(new AnnotationEntityId(annotationId, fileId), requestDate);
}
}

22
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -31,7 +30,7 @@ public class RecategorizationPersistenceService {
AnnotationEntityId id = new AnnotationEntityId(request.getAnnotationId(), fileId);
ManualRecategorizationEntity entity = recategorizationRepository.findById(id)
.orElse(ManualRecategorizationEntity.builder().id(id).build());
.orElse(buildEntity(id));
entity.setUser(request.getUser());
@ -61,6 +60,14 @@ public class RecategorizationPersistenceService {
}
private static ManualRecategorizationEntity buildEntity(AnnotationEntityId id) {
var e = new ManualRecategorizationEntity();
e.setId(id);
return e;
}
@Transactional
public void hardDelete(String fileId, String annotationId) {
@ -88,24 +95,32 @@ public class RecategorizationPersistenceService {
.orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId));
}
public List<ManualRecategorizationEntity> findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) {
if (options.getAnnotationIds().isEmpty()) {
return recategorizationRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges());
}
return recategorizationRepository.findByFileIdAndOptionsAndAnnotationIds(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges(), options.getAnnotationIds());
return recategorizationRepository.findByFileIdAndOptionsAndAnnotationIds(fileId,
options.isIncludeDeletions(),
options.isIncludeOnlyUnprocessed(),
options.isIncludeDictChanges(),
options.getAnnotationIds());
}
public int softDeleteByFileIds(List<String> fileIds, OffsetDateTime softDeletionTime) {
return recategorizationRepository.softDeleteByFileIds(fileIds, softDeletionTime);
}
public int deleteByFileIds(List<String> fileIds) {
return recategorizationRepository.deleteByFileIds(fileIds);
}
@Transactional
public void markAsProcessed(String annotationId, String fileId) {
@ -119,6 +134,7 @@ public class RecategorizationPersistenceService {
recategorizationRepository.saveAndFlush(recategorizationEntity);
}
public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) {
recategorizationRepository.undeleteByFileId(fileId, deletionTime);

24
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java
View File

@ -51,6 +51,7 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
@Param("unprocessed") boolean unprocessed,
@Param("includeDictChanges") boolean includeDictChanges);
@Query("""
select m
from ManualRedactionEntryEntity m
@ -61,27 +62,34 @@ public interface ManualRedactionRepository extends JpaRepository<ManualRedaction
and m.id.annotationId in (:annotationIds)
""")
List<ManualRedactionEntryEntity> findByFileIdAndOptionsAndAnnotationIds(@Param("fileId") String fileId,
@Param("includeDeletions") boolean includeDeletions,
@Param("unprocessed") boolean unprocessed,
@Param("includeDictChanges") boolean includeDictChanges,
@Param("annotationIds") List<String> annotationIds);
@Param("includeDeletions") boolean includeDeletions,
@Param("unprocessed") boolean unprocessed,
@Param("includeDictChanges") boolean includeDictChanges,
@Param("annotationIds") List<String> annotationIds);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.softDeletedTime = :softDeletedTime where m.id.fileId in (:fileIds) and m.softDeletedTime is null")
int softDeleteByFileIds(@Param("fileIds") List<String> fileIds,
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
int softDeleteByFileIds(@Param("fileIds") List<String> fileIds, @Param("softDeletedTime") OffsetDateTime softDeletedTime);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime")
int undeleteByFileId(@Param("fileId") String fileId,
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
int undeleteByFileId(@Param("fileId") String fileId, @Param("softDeletedTime") OffsetDateTime softDeletedTime);
@Modifying
@Query("delete ManualRedactionEntryEntity m where m.id.fileId in (:fileIds)")
int deleteByFileIds(@Param("fileIds") List<String> fileIds);
@Modifying
@Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate where m.id = :annotationEntityId")
void markAsProcessed(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("processedDate") OffsetDateTime processedDate);
@Modifying
@Query("update ManualRedactionEntryEntity e set e.requestDate = :requestDate where e.id = :annotationEntityId")
int updateRequestDateById(@Param("annotationEntityId") AnnotationEntityId annotationEntityId, @Param("requestDate") OffsetDateTime requestDate);
}

19
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -18,7 +18,6 @@ import static org.mockito.Mockito.when;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -44,7 +43,6 @@ import com.iqser.red.service.peristence.v1.server.integration.service.UserProvid
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.model.ManualChangesQueryOptions;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
@ -55,6 +53,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileMan
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.queue.SearchTermOccurrencesResponseReceiver;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionRequest;
@ -157,6 +156,9 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
@Autowired
private RecategorizationPersistenceService recategorizationPersistenceService;
@Autowired
private AddRedactionPersistenceService addRedactionPersistenceService;
@Test
public void testRemoveToDossierTemplateWithDossierDictionaryOnlyTrue() {
@ -443,7 +445,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.removeRedactionBulk(dossier.getId(),
file.getId(),
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build())).getManualAnnotationResponses()
Set.of(RemoveRedactionRequestModel.builder().annotationId("AnnotationId").removeFromDictionary(true).build()))
.getManualAnnotationResponses()
.get(0);
var dossierDictionary = internalDictionaryClient.getDictionaryForType(toTypeId(type.getType(), dossierTemplate.getId(), dossier.getId()), null);
@ -4071,10 +4074,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
assertEquals(allManualRedactions.getRecategorizations().size(), 0);
recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(),
ManualChangesQueryOptions.builder()
.includeDeletions(true)
.build());
recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(), ManualChangesQueryOptions.builder().includeDeletions(true).build());
assertEquals(recats.size(), 1);
assertEquals(annotationId, recats.get(0).getId().getAnnotationId());
assertNotNull(recats.get(0).getSoftDeletedTime());
@ -4083,10 +4083,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
manualRedactionClient.recategorizeBulk(dossier.getId(), file.getId(), recategorizationRequests);
allManualRedactions = manualRedactionClient.getManualRedactions(dossier.getId(), file.getId(), false, true);
recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(),
ManualChangesQueryOptions.builder()
.includeDeletions(true)
.build());
recats = recategorizationPersistenceService.findEntriesByFileIdAndOptions(file.getId(), ManualChangesQueryOptions.builder().includeDeletions(true).build());
assertEquals(recats.size(), 1);
assertEquals(annotationId, recats.get(0).getId().getAnnotationId());
assertNull(recats.get(0).getSoftDeletedTime());

11
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java
View File

@ -9,6 +9,7 @@ import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
@Repository
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@ -80,4 +81,14 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository<Entity
@Query(value = "{ 'positions': { $elemMatch: { 'rectangle': ?0 } }, 'engines': 'MANUAL' }", fields = "{ 'entryId': 1, 'id': 0 }")
List<String> findEntryIdsByMatchingPositionAndEngineManual(float[] rectangle);
@Query(value = """
{
'entryType': 'ENTITY',
'state': 'APPLIED',
$or: [{ 'legalBasis': null },
{ 'legalBasis': '' }
]}
""", fields = "{ 'entryId': 1, 'entityLogId': 1, 'state': 1, 'entryType': 1 , 'manualChanges': 1}")
List<EntryWithManualChangesProjection> findAppliedEntitiesWhereLegalBasisEmpty();
}

25
persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/projections/EntryWithManualChangesProjection.java Normal file
View File

@ -0,0 +1,25 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
public interface EntryWithManualChangesProjection {
String getEntryId();
String getEntityLogId();
EntryState getState();
EntryType getEntryType();
List<ManualChange> getManualChanges();
}