20260607211212

This commit is contained in:
oneao committed 2026-06-07 21:12:12 +08:00
1 parent 31dff90725
commit 4da3c88400
42 files changed
+795

No files matched your search

View File
File renamed without changes.
View File
File renamed without changes.
View File
File renamed without changes.
View File
File renamed without changes.
View File
File renamed without changes.
View File
File renamed without changes.
View File
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
View File
File renamed without changes.
File renamed without changes.
+39
View File
@@ -0,0 +1,39 @@
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
.kotlin
### IntelliJ IDEA ###
.idea/modules.xml
.idea/jarRepositories.xml
.idea/compiler.xml
.idea/libraries/
*.iws
*.iml
*.ipr
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store
+10
View File
@@ -0,0 +1,10 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Ignored default folder with query files
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
</component>
</project>
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="temurin-21" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/../../.." vcs="Git" />
</component>
</project>
@@ -0,0 +1,4 @@
#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice.
#Sun Jun 07 19:41:40 CST 2026
@default-aliyunmaven-https\://maven.aliyun.com/repository/central/.lastUpdated=1780832500060
https\://maven.aliyun.com/repository/central/.error=Could not transfer artifact org.springframework.boot\:spring-boot-starter-parent\:pom\:4.0.6 from/to aliyunmaven (https\://maven.aliyun.com/repository/central)\: Permission denied\: getsockopt
+77
View File
@@ -0,0 +1,77 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>4.0.6</version>
<relativePath/>
</parent>
<groupId>org.example</groupId>
<artifactId>springboot4-api</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>springboot4-api</name>
<description>Spring Boot 4 REST API project</description>
<properties>
<java.version>21</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<!-- Spring MVC -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webmvc</artifactId>
</dependency>
<!-- 参数校验 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<!-- OkHttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>5.1.0</version>
</dependency>
<!-- Jackson(如果IDE识别不到可显式添加) -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- Test -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<argLine>-Xshare:off -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,125 @@
package org.example;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.http.*;
import org.springframework.web.client.RestTemplate;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.util.Base64;
public class Main {
public static void main(String[] args) throws Exception {
String apiUrl = "http://localhost:8866/layout-parsing";
String pdfPath = "D:/workspace/bgd/bgd-01.pdf";
// 读取PDF
File file = new File(pdfPath);
byte[] bytes = Files.readAllBytes(file.toPath());
// Base64编码
String base64 = Base64.getEncoder().encodeToString(bytes);
// JSON请求体
ObjectMapper mapper = new ObjectMapper();
ObjectNode payload = mapper.createObjectNode();
payload.put("file", base64);
payload.put("fileType", 0);
// 打印请求大小
System.out.println("Base64 Length: " + base64.length());
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<String> request =
new HttpEntity<>(payload.toString(), headers);
ResponseEntity<String> response =
restTemplate.exchange(
apiUrl,
HttpMethod.POST,
request,
String.class
);
System.out.println("HTTP Status: "
+ response.getStatusCode());
String responseBody = response.getBody();
System.out.println("Response:");
System.out.println(responseBody);
// 解析返回结果
JsonNode root = mapper.readTree(responseBody);
JsonNode result = root.path("result");
JsonNode layoutParsingResults =
result.path("layoutParsingResults");
for (int i = 0; i < layoutParsingResults.size(); i++) {
JsonNode item = layoutParsingResults.get(i);
System.out.println(
"\n========== Page " + i + " =========="
);
JsonNode prunedResult =
item.path("prunedResult");
System.out.println(
mapper.writerWithDefaultPrettyPrinter()
.writeValueAsString(prunedResult)
);
JsonNode outputImages =
item.path("outputImages");
outputImages.fieldNames()
.forEachRemaining(imgName -> {
try {
String imgBase64 =
outputImages.get(imgName)
.asText();
byte[] imgBytes =
Base64.getDecoder()
.decode(imgBase64);
String imgPath =
imgName + "_" + i + ".jpg";
try (FileOutputStream fos =
new FileOutputStream(imgPath)) {
fos.write(imgBytes);
}
System.out.println(
"Saved image: " + imgPath
);
} catch (Exception e) {
System.err.println(
"Save image failed: "
+ e.getMessage()
);
}
});
}
}
}
@@ -0,0 +1,12 @@
package org.example;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class Springboot4ApiApplication {
public static void main(String[] args) {
SpringApplication.run(Springboot4ApiApplication.class, args);
}
}
@@ -0,0 +1,20 @@
package org.example.api;
import java.time.Instant;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api")
public class HealthController {
@GetMapping("/health")
public HealthResponse health() {
return new HealthResponse("UP", Instant.now());
}
public record HealthResponse(String status, Instant timestamp) {
}
}
@@ -0,0 +1,37 @@
package org.example.api.layout;
import jakarta.validation.Valid;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/layout-parsing")
public class LayoutParsingController {
private final LayoutParsingUseCase layoutParsingUseCase;
private final String defaultFilePath;
public LayoutParsingController(
LayoutParsingUseCase layoutParsingUseCase,
@Value("${layout.parsing.default-file-path}") String defaultFilePath
) {
this.layoutParsingUseCase = layoutParsingUseCase;
this.defaultFilePath = defaultFilePath;
}
@GetMapping
public LayoutParsingResponse parseDefault() {
LayoutParsingRequest request = LayoutParsingRequest.pdf(defaultFilePath);
return layoutParsingUseCase.parse(request);
}
@PostMapping
public LayoutParsingResponse parse(@Valid @RequestBody LayoutParsingRequest request) {
return layoutParsingUseCase.parse(request);
}
}
@@ -0,0 +1,68 @@
package org.example.api.layout;
import java.nio.file.Path;
import java.util.List;
public record LayoutParsingRequest(
String filePath,
String imagePath,
Integer fileType,
String outputDirectory,
Boolean useDocOrientationClassify,
Boolean useDocUnwarping,
Boolean useTextlineOrientation,
Boolean useSealRecognition,
Boolean useTableRecognition,
Boolean useFormulaRecognition,
Boolean useChartRecognition,
Boolean useRegionDetection,
Boolean formatBlockContent,
Boolean returnMarkdownImages,
Boolean visualize,
List<String> outputFormats
) {
public static LayoutParsingRequest pdf(String filePath) {
return new LayoutParsingRequest(
filePath,
null,
0,
null,
null,
null,
null,
null,
null,
null,
null,
null,
true,
false,
false,
null
);
}
public String resolvedFilePath() {
if (hasText(filePath)) {
return filePath;
}
if (hasText(imagePath)) {
return imagePath;
}
throw new IllegalArgumentException("filePath is required");
}
public int resolvedFileType() {
if (fileType != null) {
return fileType;
}
String fileName = Path.of(resolvedFilePath()).getFileName().toString().toLowerCase();
return fileName.endsWith(".pdf") ? 0 : 1;
}
private boolean hasText(String value) {
return value != null && !value.isBlank();
}
}
@@ -0,0 +1,23 @@
package org.example.api.layout;
import java.util.List;
import tools.jackson.databind.JsonNode;
public record LayoutParsingResponse(
String filePath,
int fileType,
List<Item> items
) {
public record Item(
JsonNode prunedResult,
String recognizedText,
String markdownText,
String savedPrunedResult,
String savedMarkdown,
String savedText,
List<String> savedImages
) {
}
}
@@ -0,0 +1,216 @@
package org.example.api.layout;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestClient;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.node.ObjectNode;
@Service
public class LayoutParsingService implements LayoutParsingUseCase {
private final RestClient restClient;
private final ObjectMapper objectMapper;
private final String apiUrl;
private final Path defaultOutputDirectory;
public LayoutParsingService(
ObjectMapper objectMapper,
@Value("${layout.parsing.api-url}") String apiUrl,
@Value("${layout.parsing.output-directory:output}") String defaultOutputDirectory
) {
this.restClient = RestClient.create();
this.objectMapper = objectMapper;
this.apiUrl = apiUrl;
this.defaultOutputDirectory = Path.of(defaultOutputDirectory);
}
@Override
public LayoutParsingResponse parse(LayoutParsingRequest request) {
Path filePath = Path.of(request.resolvedFilePath()).toAbsolutePath().normalize();
if (!Files.isRegularFile(filePath)) {
throw new IllegalArgumentException("File does not exist: " + filePath);
}
JsonNode root = callLayoutParsingApi(filePath, request);
JsonNode layoutParsingResults = root.path("result").path("layoutParsingResults");
if (!layoutParsingResults.isArray()) {
throw new IllegalStateException("Invalid layout parsing response: result.layoutParsingResults is missing");
}
Path outputDirectory = resolveOutputDirectory(request.outputDirectory());
List<LayoutParsingResponse.Item> items = new ArrayList<>();
System.out.println("Layout parsing file: " + filePath);
System.out.println("Layout parsing result count: " + layoutParsingResults.size());
for (int i = 0; i < layoutParsingResults.size(); i++) {
JsonNode item = layoutParsingResults.get(i);
JsonNode prunedResult = item.path("prunedResult");
String markdownText = item.path("markdown").path("text").asText();
String recognizedText = extractRecognizedText(prunedResult);
String savedPrunedResult = savePrunedResult(prunedResult, outputDirectory, i);
String savedMarkdown = saveTextFile(markdownText, outputDirectory, "markdown_" + i + ".md");
String savedText = saveTextFile(recognizedText, outputDirectory, "recognized_text_" + i + ".txt");
List<String> savedImages = saveOutputImages(item.path("outputImages"), outputDirectory, i);
System.out.println("Pruned Result [" + i + "]: " + prunedResult);
System.out.println("Recognized Text [" + i + "]:");
System.out.println(recognizedText);
System.out.println("Markdown Text [" + i + "]:");
System.out.println(markdownText);
System.out.println("Saved pruned result: " + savedPrunedResult);
System.out.println("Saved markdown: " + savedMarkdown);
System.out.println("Saved text: " + savedText);
savedImages.forEach(savedImage -> System.out.println("Saved image: " + savedImage));
items.add(new LayoutParsingResponse.Item(
prunedResult,
recognizedText,
markdownText,
savedPrunedResult,
savedMarkdown,
savedText,
savedImages
));
}
return new LayoutParsingResponse(filePath.toString(), request.resolvedFileType(), items);
}
private JsonNode callLayoutParsingApi(Path filePath, LayoutParsingRequest request) {
try {
String base64File = Base64.getEncoder().encodeToString(Files.readAllBytes(filePath));
ObjectNode payload = objectMapper.createObjectNode();
payload.put("file", base64File);
payload.put("fileType", request.resolvedFileType());
putIfNotNull(payload, "useDocOrientationClassify", request.useDocOrientationClassify());
putIfNotNull(payload, "useDocUnwarping", request.useDocUnwarping());
putIfNotNull(payload, "useTextlineOrientation", request.useTextlineOrientation());
putIfNotNull(payload, "useSealRecognition", request.useSealRecognition());
putIfNotNull(payload, "useTableRecognition", request.useTableRecognition());
putIfNotNull(payload, "useFormulaRecognition", request.useFormulaRecognition());
putIfNotNull(payload, "useChartRecognition", request.useChartRecognition());
putIfNotNull(payload, "useRegionDetection", request.useRegionDetection());
putIfNotNull(payload, "formatBlockContent", request.formatBlockContent());
putIfNotNull(payload, "returnMarkdownImages", request.returnMarkdownImages());
putIfNotNull(payload, "visualize", request.visualize());
if (request.outputFormats() != null && !request.outputFormats().isEmpty()) {
payload.set("outputFormats", objectMapper.valueToTree(request.outputFormats()));
}
return restClient.post()
.uri(apiUrl)
.contentType(MediaType.APPLICATION_JSON)
.body(payload)
.retrieve()
.body(JsonNode.class);
} catch (IOException e) {
throw new IllegalStateException("Failed to read file: " + filePath, e);
}
}
private void putIfNotNull(ObjectNode payload, String name, Boolean value) {
if (value != null) {
payload.put(name, value);
}
}
private Path resolveOutputDirectory(String outputDirectory) {
Path directory = StringUtils.hasText(outputDirectory)
? Path.of(outputDirectory)
: defaultOutputDirectory;
try {
Files.createDirectories(directory);
return directory.toAbsolutePath().normalize();
} catch (IOException e) {
throw new IllegalStateException("Failed to create output directory: " + directory, e);
}
}
private List<String> saveOutputImages(JsonNode outputImages, Path outputDirectory, int resultIndex) {
List<String> savedImages = new ArrayList<>();
if (!outputImages.isObject()) {
return savedImages;
}
Iterator<Entry<String, JsonNode>> fields = outputImages.properties().iterator();
while (fields.hasNext()) {
Entry<String, JsonNode> field = fields.next();
String imageName = sanitizeFileName(field.getKey());
Path outputPath = outputDirectory.resolve(imageName + "_" + resultIndex + ".jpg");
try {
byte[] imageBytes = Base64.getDecoder().decode(field.getValue().asText());
Files.write(outputPath, imageBytes);
savedImages.add(outputPath.toString());
} catch (IllegalArgumentException | IOException e) {
throw new IllegalStateException("Failed to save output image: " + outputPath, e);
}
}
return savedImages;
}
private String savePrunedResult(JsonNode prunedResult, Path outputDirectory, int resultIndex) {
Path outputPath = outputDirectory.resolve("pruned_result_" + resultIndex + ".json");
try {
Files.writeString(outputPath, prunedResult.toString());
return outputPath.toString();
} catch (IOException e) {
throw new IllegalStateException("Failed to save pruned result: " + outputPath, e);
}
}
private String saveTextFile(String content, Path outputDirectory, String fileName) {
if (!StringUtils.hasText(content)) {
return null;
}
Path outputPath = outputDirectory.resolve(fileName);
try {
Files.writeString(outputPath, content);
return outputPath.toString();
} catch (IOException e) {
throw new IllegalStateException("Failed to save text file: " + outputPath, e);
}
}
private String extractRecognizedText(JsonNode prunedResult) {
List<String> lines = new ArrayList<>();
JsonNode parsingResults = prunedResult.path("parsing_res_list");
if (parsingResults.isArray()) {
for (JsonNode block : parsingResults) {
String blockContent = block.path("block_content").asText();
if (StringUtils.hasText(blockContent)) {
lines.add(blockContent);
}
}
}
JsonNode recognizedTexts = prunedResult.path("overall_ocr_res").path("rec_texts");
if (lines.isEmpty() && recognizedTexts.isArray()) {
for (JsonNode text : recognizedTexts) {
String value = text.asText();
if (StringUtils.hasText(value)) {
lines.add(value);
}
}
}
return String.join(System.lineSeparator(), lines);
}
private String sanitizeFileName(String value) {
String sanitized = value.replaceAll("[^a-zA-Z0-9._-]", "_");
return StringUtils.hasText(sanitized) ? sanitized : "image";
}
}
@@ -0,0 +1,6 @@
package org.example.api.layout;
public interface LayoutParsingUseCase {
LayoutParsingResponse parse(LayoutParsingRequest request);
}
@@ -0,0 +1,9 @@
### Send POST request with json body
POST http://localhost:8866/layout-parsing
Content-Type: application/json
{
"file": "D:\\workspace\\bgd\\bgd-01.pdf",
"fileType": 0
}
@@ -0,0 +1,12 @@
spring:
application:
name: springboot4-api
server:
port: 8080
layout:
parsing:
api-url: http://localhost:8866/layout-parsing
default-file-path: D:/workspace/bgd/bgd-01.pdf
output-directory: output
@@ -0,0 +1,12 @@
package org.example;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class Springboot4ApiApplicationTests {
@Test
void contextLoads() {
}
}
@@ -0,0 +1,25 @@
package org.example.api;
import static org.hamcrest.Matchers.is;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest;
import org.springframework.test.web.servlet.MockMvc;
@WebMvcTest(HealthController.class)
class HealthControllerTests {
@Autowired
private MockMvc mockMvc;
@Test
void healthReturnsUp() throws Exception {
mockMvc.perform(get("/api/health"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.status", is("UP")));
}
}
@@ -0,0 +1,73 @@
package org.example.api.layout;
import static org.hamcrest.Matchers.is;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest;
import org.springframework.context.annotation.Bean;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import tools.jackson.databind.ObjectMapper;
@WebMvcTest(LayoutParsingController.class)
class LayoutParsingControllerTests {
@Autowired
private MockMvc mockMvc;
@Autowired
private ObjectMapper objectMapper;
@Test
void parseAcceptsLocalPdfPath() throws Exception {
String request = """
{
"filePath": "D:/workspace/bgd/bgd-01.pdf",
"outputDirectory": "output",
"returnMarkdownImages": false,
"visualize": false
}
""";
mockMvc.perform(post("/api/layout-parsing")
.contentType(MediaType.APPLICATION_JSON)
.content(request))
.andExpect(status().isOk())
.andExpect(jsonPath("$.filePath", is("D:/workspace/bgd/bgd-01.pdf")))
.andExpect(jsonPath("$.fileType", is(0)))
.andExpect(jsonPath("$.items[0].recognizedText", is("demo text")))
.andExpect(jsonPath("$.items[0].markdownText", is("# demo")))
.andExpect(jsonPath("$.items[0].savedPrunedResult", is("output/pruned_result_0.json")))
.andExpect(jsonPath("$.items[0].savedMarkdown", is("output/markdown_0.md")))
.andExpect(jsonPath("$.items[0].savedText", is("output/recognized_text_0.txt")))
.andExpect(jsonPath("$.items[0].savedImages[0]", is("output/table_0.jpg")));
}
@TestConfiguration
static class TestConfig {
@Bean
LayoutParsingUseCase layoutParsingUseCase(ObjectMapper objectMapper) {
return request -> new LayoutParsingResponse(
request.resolvedFilePath(),
request.resolvedFileType(),
List.of(new LayoutParsingResponse.Item(
objectMapper.createObjectNode().put("text", "demo"),
"demo text",
"# demo",
"output/pruned_result_0.json",
"output/markdown_0.md",
"output/recognized_text_0.txt",
List.of("output/table_0.jpg")
))
);
}
}
}