Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #57: 3.0 efsa readiness

Browse Source 
Merge in RED/persistence-service from 3.0-efsa-readiness to master

* commit '105f48f30894f356c936fc3cda94caf9a2328218':
  cyclic dep
  calculate flags skips some pages
  logs for analysis required
This commit is contained in:
Timo Bejan 2021-10-12 09:59:19 +02:00
commit 753eff7ea7
12 changed files with 65 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ForceRedactionRequest.java
View File

@ -16,5 +16,6 @@ public class ForceRedactionRequest {
private AnnotationStatus status;
private String legalBasis;
private String comment;
private int page;
}

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/ImageRecategorizationRequest.java
View File

@ -16,5 +16,6 @@ public class ImageRecategorizationRequest {
private AnnotationStatus status;
private String typeId;
private String comment;
private int page;
}

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/LegalBasisChangeRequest.java
View File

@ -16,5 +16,6 @@ public class LegalBasisChangeRequest {
private AnnotationStatus status;
private String legalBasis;
private String comment;
private int page;
}

1
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/annotations/RemoveRedactionRequest.java
View File

@ -16,5 +16,6 @@ public class RemoveRedactionRequest {
private AnnotationStatus status;
private boolean removeFromDictionary;
private String comment;
private int page;
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/IdRemovalEntity.java
View File

@ -32,6 +32,8 @@ public class IdRemovalEntity {
private OffsetDateTime processedDate;
@Column
private OffsetDateTime softDeletedTime;
@Column
private int page;
@ManyToOne
private FileEntity fileStatus;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualForceRedactionEntity.java
View File

@ -32,6 +32,8 @@ public class ManualForceRedactionEntity {
private OffsetDateTime processedDate;
@Column
private OffsetDateTime softDeletedTime;
@Column
private int page;
@ManyToOne
private FileEntity fileStatus;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualImageRecategorizationEntity.java
View File

@ -32,6 +32,8 @@ public class ManualImageRecategorizationEntity {
private OffsetDateTime processedDate;
@Column
private OffsetDateTime softDeletedTime;
@Column
private int page;
@ManyToOne
private FileEntity fileStatus;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualLegalBasisChangeEntity.java
View File

@ -33,6 +33,8 @@ public class ManualLegalBasisChangeEntity {
private OffsetDateTime processedDate;
@Column
private OffsetDateTime softDeletedTime;
@Column
private int page;
@ManyToOne
private FileEntity fileStatus;

37
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ManualRedactionController.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.peristence.v1.server.controller;
import com.iqser.red.service.peristence.v1.server.service.AnalysisFlagsCalculationService;
import com.iqser.red.service.peristence.v1.server.service.ManualRedactionService;
import com.iqser.red.service.peristence.v1.server.utils.ManualImageRecategorizationMapper;
import com.iqser.red.service.peristence.v1.server.utils.ManualRedactionMapper;
@ -19,7 +18,6 @@ import static com.iqser.red.service.persistence.management.v1.processor.utils.Ma
public class ManualRedactionController implements ManualRedactionResource {
private final ManualRedactionService manualRedactionService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
@Override
@ -27,9 +25,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(FILE_ID) String fileId,
@RequestBody AddRedactionRequest addRedactionRequest) {
var resp = manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return manualRedactionService.addAddRedaction(dossierId, fileId, addRedactionRequest);
}
@ -38,9 +34,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(FILE_ID) String fileId,
@RequestBody RemoveRedactionRequest removeRedactionRequest) {
var resp = manualRedactionService.addRemoveRedaction(dossierId, fileId, removeRedactionRequest);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return manualRedactionService.addRemoveRedaction(dossierId, fileId, removeRedactionRequest);
}
@ -49,9 +43,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(FILE_ID) String fileId,
@RequestBody ForceRedactionRequest forceRedactionRequest) {
var resp = manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequest);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return manualRedactionService.addForceRedaction(dossierId, fileId, forceRedactionRequest);
}
@ -60,9 +52,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(FILE_ID) String fileId,
@RequestBody LegalBasisChangeRequest legalBasisChangeRequest) {
var resp = manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequest);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return manualRedactionService.addLegalBasisChange(dossierId, fileId, legalBasisChangeRequest);
}
@ -71,9 +61,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(FILE_ID) String fileId,
@RequestBody ImageRecategorizationRequest imageRecategorizationRequest) {
var resp = manualRedactionService.addImageRecategorization(dossierId, fileId, imageRecategorizationRequest);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return manualRedactionService.addImageRecategorization(dossierId, fileId, imageRecategorizationRequest);
}
@ -82,9 +70,7 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId,
@RequestBody CommentRequest comment) {
var resp = convert(manualRedactionService.addComment(fileId, annotationId, comment), Comment.class);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return resp;
return convert(manualRedactionService.addComment(fileId, annotationId, comment), Comment.class);
}
@ -138,7 +124,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId) {
manualRedactionService.deleteAddRedaction(dossierId, fileId, annotationId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -147,7 +132,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId) {
manualRedactionService.deleteRemoveRedaction(dossierId, fileId, annotationId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -156,7 +140,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId) {
manualRedactionService.deleteForceRedaction(dossierId, fileId, annotationId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -165,7 +148,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId) {
manualRedactionService.deleteLegalBasisChange(dossierId, fileId, annotationId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -175,7 +157,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@PathVariable(ANNOTATION_ID) String annotationId) {
manualRedactionService.deleteImageRecategorization(dossierId, fileId, annotationId);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -183,7 +164,6 @@ public class ManualRedactionController implements ManualRedactionResource {
public void deleteComment(@PathVariable(FILE_ID) String fileId, @PathVariable(COMMENT_ID) long commentId) {
manualRedactionService.deleteComment(fileId, commentId);
// TODO analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -194,7 +174,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@RequestBody JSONPrimitive<AnnotationStatus> updateStatusRequest) {
manualRedactionService.updateAddRedactionStatus(dossierId, fileId, annotationId, updateStatusRequest.getValue());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -205,7 +184,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@RequestBody JSONPrimitive<AnnotationStatus> updateStatusRequest) {
manualRedactionService.updateRemoveRedactionStatus(dossierId, fileId, annotationId, updateStatusRequest.getValue());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -216,7 +194,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@RequestBody JSONPrimitive<AnnotationStatus> updateStatusRequest) {
manualRedactionService.updateForceRedactionStatus(dossierId, fileId, annotationId, updateStatusRequest.getValue());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -226,7 +203,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@RequestBody JSONPrimitive<AnnotationStatus> updateStatusRequest) {
manualRedactionService.updateLegalBasisChangeStatus(dossierId, fileId, annotationId, updateStatusRequest.getValue());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -237,7 +213,6 @@ public class ManualRedactionController implements ManualRedactionResource {
@RequestBody JSONPrimitive<AnnotationStatus> updateStatusRequest) {
manualRedactionService.updateImageRecategorizationStatus(dossierId, fileId, annotationId, updateStatusRequest.getValue());
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}

11
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
@ -13,6 +14,7 @@ import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@ -28,8 +30,13 @@ public class AnalysisFlagsCalculationService {
@Async
public void calculateFlags(String dossierId, String fileId) {
this.calculateFlags(dossierId,fileId, Sets.newHashSet());
}
long startTime = System.nanoTime();
@Async
public void calculateFlags(String dossierId, String fileId, Set<Integer> pagesToSkip) {
long startTime = System.currentTimeMillis();
var file = fileStatusPersistenceService.getStatus(fileId);
var redactionLog = redactionLogService.getRedactionLog(dossierId, fileId, true);
@ -89,6 +96,8 @@ public class AnalysisFlagsCalculationService {
}
viewedPagesToReset.removeAll(pagesToSkip);
viewedPagesPersistenceService.resetViewedPages(fileId, file.getCurrentReviewer(), viewedPagesToReset);
fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates);

37
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.peristence.v1.server.service;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.controller.DictionaryController;
@ -41,6 +42,7 @@ public class ManualRedactionService {
private final LegalBasisChangePersistenceService legalBasisChangePersistenceService;
private final FileStatusService fileStatusService;
private final ManualRedactionProviderService manualRedactionProviderService;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final StopwordService stopwordService;
private final HashFunction hashFunction = Hashing.murmur3_128();
@ -85,6 +87,10 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
if (!addRedactionRequest.getPositions().isEmpty()) {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(addRedactionRequest.getPositions().get(0).getPage()));
}
return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build();
}
@ -113,6 +119,10 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
if (!removeRedactionRequest.isRemoveFromDictionary()) {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(removeRedactionRequest.getPage()));
}
return ManualAddResponse.builder()
.annotationId(removeRedactionRequest.getAnnotationId())
.commentId(commentId)
@ -141,6 +151,8 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(forceRedactionRequest.getPage()));
return ManualAddResponse.builder()
.annotationId(forceRedactionRequest.getAnnotationId())
.commentId(commentId)
@ -169,6 +181,8 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateLastManualRedaction(fileId, now);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(legalBasisChangeRequest.getPage()));
return ManualAddResponse.builder()
.annotationId(legalBasisChangeRequest.getAnnotationId())
.commentId(commentId)
@ -198,6 +212,8 @@ public class ManualRedactionService {
fileStatusService.setStatusReprocess(dossierId, fileId, 2);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
return ManualAddResponse.builder()
.annotationId(imageRecategorizationRequest.getAnnotationId())
.commentId(commentId)
@ -210,6 +226,8 @@ public class ManualRedactionService {
var createdComment = addComment(fileId, annotationId, commentRequest.getText(), commentRequest.getUser());
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
fileStatusPersistenceService.updateHasComments(fileId, true);
return createdComment;
}
@ -266,6 +284,10 @@ public class ManualRedactionService {
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
}
if (!addRedaction.getPositions().isEmpty()) {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(addRedaction.getPositions().get(0).getPage()));
}
}
@ -285,6 +307,7 @@ public class ManualRedactionService {
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(removeRedaction.getPage()));
}
@ -300,6 +323,7 @@ public class ManualRedactionService {
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(forceRedaction.getPage()));
}
@ -314,6 +338,7 @@ public class ManualRedactionService {
} else {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(legalBasisChange.getPage()));
}
@ -330,6 +355,7 @@ public class ManualRedactionService {
fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now());
fileStatusService.setStatusReprocess(dossierId, fileId, 2);
}
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -340,6 +366,7 @@ public class ManualRedactionService {
// update indicator
fileStatusPersistenceService.updateHasComments(fileId, commentPersistenceService.fileHasComments(fileId));
}
@ -379,6 +406,8 @@ public class ManualRedactionService {
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(idRemoval.getPage()));
}
private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) {
@ -396,6 +425,7 @@ public class ManualRedactionService {
forceRedactionPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
@ -405,16 +435,17 @@ public class ManualRedactionService {
legalBasisChangePersistenceService.updateStatus(fileId, annotationId, annotationStatus);
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId);
}
public void updateImageRecategorizationStatus(String dossierId, String fileId, String annotationId,
AnnotationStatus annotationStatus) {
ManualImageRecategorizationEntity imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
if (annotationStatus.equals(AnnotationStatus.DECLINED)) {
ManualImageRecategorizationEntity imageRecategorization = recategorizationPersistenceService.findRecategorization(fileId, annotationId);
// if it was previously approved, revert the delete
if (imageRecategorization.getStatus() == AnnotationStatus.APPROVED) {
fileStatusService.setStatusReprocess(dossierId, fileId, 2);
@ -427,6 +458,7 @@ public class ManualRedactionService {
recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus);
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(imageRecategorization.getPage()));
}
@ -451,6 +483,9 @@ public class ManualRedactionService {
boolean hasSuggestions = calculateHasSuggestions(fileId);
fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions);
if (!manualRedactionEntry.getPositions().isEmpty()) {
analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(manualRedactionEntry.getPositions().get(0).getPage()));
}
}

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/RedactionLogService.java
View File

@ -14,7 +14,7 @@ import org.springframework.stereotype.Service;
public class RedactionLogService {
private final FileManagementStorageService fileManagementStorageService;
private final ManualRedactionService manualRedactionService;
private final ManualRedactionProviderService manualRedactionService;
private final RedactionClient redactionClient;
private final DossierPersistenceService dossierPersistenceService;
private final FileStatusService fileStatusService;