Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Cycle hotfix #268

Merged
dominique.eiflaender1 merged 2 commits from cycleHotfix into master 2023-12-14 14:04:40 +01:00
Conversation 4 Commits 2 Files Changed 7 +101 -37
7 changed files with 101 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
persistence-service-v1/persistence-service-external-api-v1/build.gradle.kts
View File

@ -4,10 +4,22 @@ plugins {
dependencies {
api(project(":persistence-service-internal-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0")
api("com.iqser.red.service:redaction-service-api-v1:4.196.0")
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0")
api("com.iqser.red.service:search-service-api-v1:2.71.0")
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.4")
api("com.google.guava:guava:31.1-jre")
api("org.springframework.boot:spring-boot-starter-security:3.1.3")

20
persistence-service-v1/persistence-service-external-api-v2/build.gradle.kts
View File

@ -6,10 +6,22 @@ plugins {
dependencies {
api(project(":persistence-service-external-api-v1"))
api(project(":persistence-service-internal-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0")
api("com.iqser.red.service:redaction-service-api-v1:4.196.0")
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0")
api("com.iqser.red.service:search-service-api-v1:2.71.0")
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0")
api("com.google.guava:guava:31.1-jre")
api("org.springframework.boot:spring-boot-starter-security:3.1.3")

1
persistence-service-v1/persistence-service-internal-api-impl-v1/build.gradle.kts
View File

@ -5,6 +5,7 @@ plugins {
dependencies {
api(project(":persistence-service-processor-v1"))
api(project(":persistence-service-external-api-v1"))
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
}

32
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -8,18 +8,37 @@ val springBootStarterVersion = "3.1.5"
dependencies {
api(project(":persistence-service-external-api-v1"))
api(project(":persistence-service-internal-api-v1"))
api(project(":persistence-service-shared-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-service-api-v1:4.196.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:layoutparser-service-internal-api:0.74.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.iqser.red.service:search-service-api-v1:2.71.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight.service:ocr-service-api:4.5.0") {
exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1")
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:database-tenant-commons:0.20.0")
api("com.knecon.fforesight:keycloak-commons:0.24.0")
api("com.knecon.fforesight:tracing-commons:0.2.0")
api("com.knecon.fforesight:swagger-commons:0.5.0")
api("com.iqser.red.service:pdftron-redaction-service-api-v1:4.38.0")
api("com.iqser.red.service:redaction-service-api-v1:4.196.0")
api("com.iqser.red.service:redaction-report-service-api-v1:4.36.0")
api("com.knecon.fforesight:layoutparser-service-internal-api:0.74.0")
api("com.iqser.red.service:search-service-api-v1:2.71.0")
api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0")
api("com.knecon.fforesight.service:ocr-service-api:4.5.0")
api("org.springframework.security:spring-security-acl:6.0.2")
api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}")
@ -38,7 +57,6 @@ dependencies {
api("com.opencsv:opencsv:5.4")
api("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
api("commons-validator:commons-validator:1.7")
api("com.iqser.red.service:persistence-service-shared-api-v1:+")
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
testImplementation("org.testcontainers:postgresql:1.17.1")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadService.java
View File

@ -27,6 +27,7 @@ import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -41,6 +42,7 @@ public class UploadService {
private final AuditPersistenceService auditPersistenceService;
private final FileAttributesManagementService fileAttributesManagementService;
private final StorageService storageService;
private final ObservationRegistry observationRegistry;
@PreAuthorize("hasAuthority('" + WRITE_FILE_ATTRIBUTES + "')")
@ -67,6 +69,8 @@ public class UploadService {
dossierManagementService.getDossierById(dossierId, false, false);
var fileId = generateFileId(fileName, dossierId);
addFileIdToTrace(fileId);
var storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.UNTOUCHED);
try {
@ -88,7 +92,6 @@ public class UploadService {
.build());
return FileUploadResult.builder().fileIds(Collections.singletonList(fileId)).build();
}
@ -97,4 +100,11 @@ public class UploadService {
return hashFunction.hashBytes((fileName + dossierId).getBytes(StandardCharsets.UTF_8)).toString();
}
private void addFileIdToTrace(String fileId){
if(observationRegistry.getCurrentObservation() != null){
observationRegistry.getCurrentObservation().highCardinalityKeyValue("fileId", fileId);
}
}
}

40
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java
View File

@ -17,8 +17,9 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUti
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.UpdateDetailsRequest;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@ -32,6 +33,7 @@ public class AutomaticAnalysisJob implements Job {
private final FileManagementServiceSettings fileManagementServiceSettings;
private final FileStatusService fileStatusService;
private final TenantProvider tenantProvider;
private final ObservationRegistry observationRegistry;
@Setter
private boolean schedulingStopped;
@ -79,13 +81,7 @@ public class AutomaticAnalysisJob implements Job {
var next = allStatusesIterator.next();
// in case the file doesn't have numberOfPages set, we assume an average.
if (next.isFullAnalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), next.getFilename());
fileStatusService.setStatusFullReprocess(next.getDossierId(), next.getId(), false, false);
} else if (next.isReanalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), next.getFilename());
fileStatusService.setStatusReprocess(next.getDossierId(), next.getId(), false);
}
reanalyseFile(next);
queuedFiles++;
}
@ -99,17 +95,41 @@ public class AutomaticAnalysisJob implements Job {
}
private void reanalyseFile(FileModel file) {
Observation.createNotStarted("AutomaticAnalysisJob", observationRegistry)
.contextualName("scheduled-analysis")
.highCardinalityKeyValue("dossierTemplateId", file.getDossierTemplateId())
.highCardinalityKeyValue("dossierId", file.getDossierId())
.highCardinalityKeyValue("fileId", file.getId())
.lowCardinalityKeyValue("tenantId", TenantContext.getTenantId())
.observe(() -> {
if (file.isFullAnalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic full analysis! ", TenantContext.getTenantId(), file.getFilename());
fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, false);
} else if (file.isReanalysisRequired()) {
log.info("[Tenant:{}] Queued file: {} for automatic reanalysis! ", TenantContext.getTenantId(), file.getFilename());
fileStatusService.setStatusReprocess(file.getDossierId(), file.getId(), false);
}
});
}
private List<FileModel> getAllRelevantStatuses() {
return fileStatusService.getAllRelevantStatusesForReanalysisScheduler();
}
public void stopForTenant(String tenantId){
public void stopForTenant(String tenantId) {
stoppedTenants.add(tenantId);
}
public void startForTenant(String tenantId){
public void startForTenant(String tenantId) {
stoppedTenants.remove(tenantId);
}

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -132,13 +132,7 @@ public class Application implements ApplicationContextAware {
@Override
public KeyValues getLowCardinalityKeyValues(ServerRequestObservationContext context) {
// Make sure that KeyValues entries are already sorted by name for better performance
return super.getLowCardinalityKeyValues(context)
.and(getValueFromPathVariableOrRequestParam(context, "dossierId"),
getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"),
getValueFromPathVariableOrRequestParam(context, "fileId"),
KeyValue.of("kubernetes.namespace", namespace),
KeyValue.of("service.version", version),
tenantId(context));
return super.getLowCardinalityKeyValues(context).and(KeyValue.of("kubernetes.namespace", namespace), KeyValue.of("service.version", version), tenantId(context));
}
@ -148,10 +142,7 @@ public class Application implements ApplicationContextAware {
return super.getHighCardinalityKeyValues(context)
.and(getValueFromPathVariableOrRequestParam(context, "dossierId"),
getValueFromPathVariableOrRequestParam(context, "dossierTemplateId"),
getValueFromPathVariableOrRequestParam(context, "fileId"),
KeyValue.of("kubernetes.namespace", namespace),
KeyValue.of("service.version", version),
tenantId(context));
getValueFromPathVariableOrRequestParam(context, "fileId"));
}