javier/Stirling-PDF
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into expose-port

Browse Source
This commit is contained in:
sbplat
2023-12-29 17:01:49 -05:00
committed by GitHub
parent 94aba370e0 80a59205fa
commit 036fd711f9
60 changed files with 1757 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/stirling/software/SPDF/SPdfApplication.java
View File

@@ -8,13 +8,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.EnableScheduling;
import jakarta.annotation.PostConstruct;
import stirling.software.SPDF.config.ConfigInitializer;
import stirling.software.SPDF.utils.GeneralUtils;
@SpringBootApplication
//@EnableScheduling
@EnableScheduling
public class SPdfApplication {
@Autowired

6
src/main/java/stirling/software/SPDF/config/AppConfig.java
View File

@@ -41,12 +41,16 @@ public class AppConfig {
return (defaultNavBar != null) ? defaultNavBar : "Stirling PDF";
}
@Bean(name = "enableAlphaFunctionality")
public boolean enableAlphaFunctionality() {
return applicationProperties.getSystem().getEnableAlphaFunctionality() != null ? applicationProperties.getSystem().getEnableAlphaFunctionality() : false;
}
@Bean(name = "rateLimit")
public boolean rateLimit() {
String appName = System.getProperty("rateLimit");
if (appName == null)
appName = System.getenv("rateLimit");
System.out.println("rateLimit=" + appName);
return (appName != null) ? Boolean.valueOf(appName) : false;
}

48
src/main/java/stirling/software/SPDF/config/MetricsFilter.java
View File

@@ -16,33 +16,35 @@ import jakarta.servlet.http.HttpServletResponse;
@Component
public class MetricsFilter extends OncePerRequestFilter {
private final MeterRegistry meterRegistry;
private final MeterRegistry meterRegistry;
@Autowired
public MetricsFilter(MeterRegistry meterRegistry) {
this.meterRegistry = meterRegistry;
}
@Autowired
public MetricsFilter(MeterRegistry meterRegistry) {
this.meterRegistry = meterRegistry;
}
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
String uri = request.getRequestURI();
//System.out.println("uri="+uri + ", method=" + request.getMethod() );
// Ignore static resources
if (!(uri.startsWith("/js") || uri.startsWith("api-docs") || uri.endsWith("robots.txt") || uri.startsWith("/images") || uri.endsWith(".png") || uri.endsWith(".ico") || uri.endsWith(".css") || uri.endsWith(".svg")|| uri.endsWith(".js") || uri.contains("swagger") || uri.startsWith("/api"))) {
Counter counter = Counter.builder("http.requests")
.tag("uri", uri)
.tag("method", request.getMethod())
.register(meterRegistry);
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
String uri = request.getRequestURI();
counter.increment();
//System.out.println("Counted");
}
// System.out.println("uri="+uri + ", method=" + request.getMethod() );
// Ignore static resources
if (!(uri.startsWith("/js") || uri.startsWith("/v1/api-docs") || uri.endsWith("robots.txt")
|| uri.startsWith("/images") || uri.startsWith("/images")|| uri.endsWith(".png") || uri.endsWith(".ico") || uri.endsWith(".css") || uri.endsWith(".map")
|| uri.endsWith(".svg") || uri.endsWith(".js") || uri.contains("swagger")
|| uri.startsWith("/api/v1/info") || uri.startsWith("/site.webmanifest") || uri.startsWith("/fonts") || uri.startsWith("/pdfjs") )) {
Counter counter = Counter.builder("http.requests").tag("uri", uri).tag("method", request.getMethod())
.register(meterRegistry);
filterChain.doFilter(request, response);
}
counter.increment();
// System.out.println("Counted");
}
filterChain.doFilter(request, response);
}
}

35
src/main/java/stirling/software/SPDF/config/OpenApiConfig.java
View File

@@ -1,27 +1,42 @@
package stirling.software.SPDF.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.security.SecurityRequirement;
import io.swagger.v3.oas.models.security.SecurityScheme;
import stirling.software.SPDF.model.ApplicationProperties;
@Configuration
public class OpenApiConfig {
@Autowired
ApplicationProperties applicationProperties;
@Bean
public OpenAPI customOpenAPI() {
String version = getClass().getPackage().getImplementationVersion();
if (version == null) {
version = "1.0.0"; // default version if all else fails
}
String version = getClass().getPackage().getImplementationVersion();
if (version == null) {
version = "1.0.0"; // default version if all else fails
}
SecurityScheme apiKeyScheme = new SecurityScheme().type(SecurityScheme.Type.APIKEY).in(SecurityScheme.In.HEADER)
.name("X-API-KEY");
if (!applicationProperties.getSecurity().getEnableLogin()) {
return new OpenAPI().components(new Components())
.info(new Info().title("Stirling PDF API").version(version).description(
"API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here."));
} else {
return new OpenAPI().components(new Components().addSecuritySchemes("apiKey", apiKeyScheme))
.info(new Info().title("Stirling PDF API").version(version).description(
"API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here."))
.addSecurityItem(new SecurityRequirement().addList("apiKey"));
}
return new OpenAPI().components(new Components()).info(
new Info().title("Stirling PDF API").version(version).description("API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here."));
}
}
}

27
src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationFailureHandler.java
View File

@@ -2,27 +2,46 @@ package stirling.software.SPDF.config.security;
import java.io.IOException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.authentication.LockedException;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler;
import org.springframework.stereotype.Component;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
@Component
public class CustomAuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler {
@Autowired
private final LoginAttemptService loginAttemptService;
@Autowired
public CustomAuthenticationFailureHandler(LoginAttemptService loginAttemptService) {
this.loginAttemptService = loginAttemptService;
}
@Override
public void onAuthenticationFailure(HttpServletRequest request, HttpServletResponse response, AuthenticationException exception)
throws IOException, ServletException {
String ip = request.getRemoteAddr();
logger.error("Failed login attempt from IP: " + ip);
if (exception.getClass().isAssignableFrom(BadCredentialsException.class)) {
setDefaultFailureUrl("/login?error=badcredentials");
} else if (exception.getClass().isAssignableFrom(LockedException.class)) {
String username = request.getParameter("username");
if(loginAttemptService.loginAttemptCheck(username)) {
setDefaultFailureUrl("/login?error=locked");
} else {
if (exception.getClass().isAssignableFrom(BadCredentialsException.class)) {
setDefaultFailureUrl("/login?error=badcredentials");
} else if (exception.getClass().isAssignableFrom(LockedException.class)) {
setDefaultFailureUrl("/login?error=locked");
}
}
super.onAuthenticationFailure(request, response, exception);
}
}

44
src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationSuccessHandler.java Normal file
View File

@@ -0,0 +1,44 @@
package stirling.software.SPDF.config.security;
import java.io.IOException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.Authentication;
import org.springframework.security.web.authentication.SavedRequestAwareAuthenticationSuccessHandler;
import org.springframework.security.web.savedrequest.SavedRequest;
import org.springframework.stereotype.Component;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import stirling.software.SPDF.utils.RequestUriUtils;
@Component
public class CustomAuthenticationSuccessHandler extends SavedRequestAwareAuthenticationSuccessHandler {
@Autowired
private LoginAttemptService loginAttemptService;
@Override
public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws ServletException, IOException {
String username = request.getParameter("username");
loginAttemptService.loginSucceeded(username);
// Get the saved request
HttpSession session = request.getSession(false);
SavedRequest savedRequest = session != null ? (SavedRequest) session.getAttribute("SPRING_SECURITY_SAVED_REQUEST") : null;
if (savedRequest != null && !RequestUriUtils.isStaticResource(savedRequest.getRedirectUrl())) {
// Redirect to the original destination
super.onAuthenticationSuccess(request, response, authentication);
} else {
// Redirect to the root URL (considering context path)
getRedirectStrategy().sendRedirect(request, response, "/");
}
//super.onAuthenticationSuccess(request, response, authentication);
}
}

9
src/main/java/stirling/software/SPDF/config/security/CustomUserDetailsService.java
View File

@@ -5,6 +5,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.LockedException;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.userdetails.UserDetails;
@@ -22,12 +23,18 @@ public class CustomUserDetailsService implements UserDetailsService {
@Autowired
private UserRepository userRepository;
@Autowired
private LoginAttemptService loginAttemptService;
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
User user = userRepository.findByUsername(username)
.orElseThrow(() -> new UsernameNotFoundException("No user found with username: " + username));
if (loginAttemptService.isBlocked(username)) {
throw new LockedException("Your account has been locked due to too many failed login attempts.");
}
return new org.springframework.security.core.userdetails.User(
user.getUsername(),
user.getPassword(),

7
src/main/java/stirling/software/SPDF/config/security/FirstLoginFilter.java
View File

@@ -15,6 +15,7 @@ import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import stirling.software.SPDF.model.User;
import stirling.software.SPDF.utils.RequestUriUtils;
@Component
public class FirstLoginFilter extends OncePerRequestFilter {
@@ -28,11 +29,7 @@ public class FirstLoginFilter extends OncePerRequestFilter {
String method = request.getMethod();
String requestURI = request.getRequestURI();
// Check if the request is for static resources
boolean isStaticResource = requestURI.startsWith("/css/")
|| requestURI.startsWith("/js/")
|| requestURI.startsWith("/images/")
|| requestURI.startsWith("/public/")
|| requestURI.endsWith(".svg");
boolean isStaticResource = RequestUriUtils.isStaticResource(requestURI);
// If it's a static resource, just continue the filter chain and skip the logic below
if (isStaticResource) {

65
src/main/java/stirling/software/SPDF/config/security/IPRateLimitingFilter.java Normal file
View File

@@ -0,0 +1,65 @@
package stirling.software.SPDF.config.security;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import jakarta.servlet.Filter;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.ServletRequest;
import jakarta.servlet.ServletResponse;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.SPDF.utils.RequestUriUtils;
public class IPRateLimitingFilter implements Filter {
private final ConcurrentHashMap<String, AtomicInteger> requestCounts = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, AtomicInteger> getCounts = new ConcurrentHashMap<>();
private final int maxRequests;
private final int maxGetRequests;
public IPRateLimitingFilter(int maxRequests, int maxGetRequests) {
this.maxRequests = maxRequests;
this.maxGetRequests = maxGetRequests;
}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
if (request instanceof HttpServletRequest) {
HttpServletRequest httpRequest = (HttpServletRequest) request;
String method = httpRequest.getMethod();
String requestURI = httpRequest.getRequestURI();
// Check if the request is for static resources
boolean isStaticResource = RequestUriUtils.isStaticResource(requestURI);
// If it's a static resource, just continue the filter chain and skip the logic below
if (isStaticResource) {
chain.doFilter(request, response);
return;
}
String clientIp = request.getRemoteAddr();
requestCounts.computeIfAbsent(clientIp, k -> new AtomicInteger(0));
if (!"GET".equalsIgnoreCase(method)) {
if (requestCounts.get(clientIp).incrementAndGet() > maxRequests) {
// Handle limit exceeded (e.g., send error response)
response.getWriter().write("Rate limit exceeded");
return;
}
} else {
if (requestCounts.get(clientIp).incrementAndGet() > maxGetRequests) {
// Handle limit exceeded (e.g., send error response)
response.getWriter().write("GET Rate limit exceeded");
return;
}
}
}
chain.doFilter(request, response);
}
public void resetRequestCounts() {
requestCounts.clear();
getCounts.clear();
}
}

6
src/main/java/stirling/software/SPDF/config/security/InitialSecuritySetup.java
View File

@@ -37,8 +37,10 @@ public class InitialSecuritySetup {
initialPassword = "stirling";
userService.saveUser(initialUsername, initialPassword, Role.ADMIN.getRoleId(), true);
}
}
if(!userService.usernameExists(Role.INTERNAL_API_USER.getRoleId())) {
userService.saveUser(Role.INTERNAL_API_USER.getRoleId(), UUID.randomUUID().toString(), Role.INTERNAL_API_USER.getRoleId());
userService.addApiKeyToUser(Role.INTERNAL_API_USER.getRoleId());
}
}

56
src/main/java/stirling/software/SPDF/config/security/LoginAttemptService.java Normal file
View File

@@ -0,0 +1,56 @@
package stirling.software.SPDF.config.security;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.AttemptCounter;
@Service
public class LoginAttemptService {
@Autowired
ApplicationProperties applicationProperties;
private int MAX_ATTEMPTS;
private long ATTEMPT_INCREMENT_TIME;
@PostConstruct
public void init() {
MAX_ATTEMPTS = applicationProperties.getSecurity().getLoginAttemptCount();
ATTEMPT_INCREMENT_TIME = TimeUnit.MINUTES.toMillis(applicationProperties.getSecurity().getLoginResetTimeMinutes());
}
private final ConcurrentHashMap<String, AttemptCounter> attemptsCache = new ConcurrentHashMap<>();
public void loginSucceeded(String key) {
attemptsCache.remove(key);
}
public boolean loginAttemptCheck(String key) {
attemptsCache.compute(key, (k, attemptCounter) -> {
if (attemptCounter == null || attemptCounter.shouldReset(ATTEMPT_INCREMENT_TIME)) {
return new AttemptCounter();
} else {
attemptCounter.increment();
return attemptCounter;
}
});
return attemptsCache.get(key).getAttemptCount() >= MAX_ATTEMPTS;
}
public boolean isBlocked(String key) {
AttemptCounter attemptCounter = attemptsCache.get(key);
if (attemptCounter != null) {
return attemptCounter.getAttemptCount() >= MAX_ATTEMPTS;
}
return false;
}
}

18
src/main/java/stirling/software/SPDF/config/security/RateLimitResetScheduler.java Normal file
View File

@@ -0,0 +1,18 @@
package stirling.software.SPDF.config.security;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
public class RateLimitResetScheduler {
private final IPRateLimitingFilter rateLimitingFilter;
public RateLimitResetScheduler(IPRateLimitingFilter rateLimitingFilter) {
this.rateLimitingFilter = rateLimitingFilter;
}
@Scheduled(cron = "0 0 0 * * MON") // At 00:00 every Monday TODO: configurable
public void resetRateLimit() {
rateLimitingFilter.resetRequestCounts();
}
}

42
src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java
View File

@@ -6,7 +6,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.dao.DaoAuthenticationProvider;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.core.userdetails.UserDetailsService;
@@ -15,12 +15,13 @@ import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.authentication.rememberme.PersistentTokenRepository;
import org.springframework.security.web.savedrequest.NullRequestCache;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
import stirling.software.SPDF.repository.JPATokenRepositoryImpl;
@Configuration
@EnableWebSecurity()
@EnableGlobalMethodSecurity(prePostEnabled = true)
@EnableMethodSecurity
public class SecurityConfiguration {
@Autowired
@@ -40,6 +41,11 @@ public class SecurityConfiguration {
@Autowired
private UserAuthenticationFilter userAuthenticationFilter;
@Autowired
private LoginAttemptService loginAttemptService;
@Autowired
private FirstLoginFilter firstLoginFilter;
@@ -51,14 +57,18 @@ public class SecurityConfiguration {
if(loginEnabledValue) {
http.csrf(csrf -> csrf.disable());
http.addFilterBefore(rateLimitingFilter(), UsernamePasswordAuthenticationFilter.class);
http.addFilterAfter(firstLoginFilter, UsernamePasswordAuthenticationFilter.class);
http
.formLogin(formLogin -> formLogin
.loginPage("/login")
.successHandler(new CustomAuthenticationSuccessHandler())
.defaultSuccessUrl("/")
.failureHandler(new CustomAuthenticationFailureHandler())
.failureHandler(new CustomAuthenticationFailureHandler(loginAttemptService))
.permitAll()
)
).requestCache(requestCache -> requestCache
.requestCache(new NullRequestCache())
)
.logout(logout -> logout
.logoutRequestMatcher(new AntPathRequestMatcher("/logout"))
.logoutSuccessUrl("/login?logout=true")
@@ -70,8 +80,19 @@ public class SecurityConfiguration {
.tokenValiditySeconds(1209600) // 2 weeks
)
.authorizeHttpRequests(authz -> authz
.requestMatchers(req -> req.getRequestURI().startsWith("/login") || req.getRequestURI().endsWith(".svg") || req.getRequestURI().startsWith("/register") || req.getRequestURI().startsWith("/error") || req.getRequestURI().startsWith("/images/") || req.getRequestURI().startsWith("/public/") || req.getRequestURI().startsWith("/css/") || req.getRequestURI().startsWith("/js/"))
.permitAll()
.requestMatchers(req -> {
String uri = req.getRequestURI();
String contextPath = req.getContextPath();
// Remove the context path from the URI
String trimmedUri = uri.startsWith(contextPath) ? uri.substring(contextPath.length()) : uri;
return trimmedUri.startsWith("/login") || trimmedUri.endsWith(".svg") ||
trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") ||
trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") ||
trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/");
}
).permitAll()
.anyRequest().authenticated()
)
.userDetailsService(userDetailsService)
@@ -84,8 +105,17 @@ public class SecurityConfiguration {
}
return http.build();
}
@Bean
public IPRateLimitingFilter rateLimitingFilter() {
int maxRequestsPerIp = 1000000; // Example limit TODO add config level
return new IPRateLimitingFilter(maxRequestsPerIp, maxRequestsPerIp);
}
@Bean
public DaoAuthenticationProvider authenticationProvider() {

26
src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java
View File

@@ -74,15 +74,17 @@ public class UserAuthenticationFilter extends OncePerRequestFilter {
// If we still don't have any authentication, deny the request
if (authentication == null || !authentication.isAuthenticated()) {
String method = request.getMethod();
if ("GET".equalsIgnoreCase(method) && !"/login".equals(requestURI)) {
response.sendRedirect("/login"); // redirect to the login page
String contextPath = request.getContextPath();
if ("GET".equalsIgnoreCase(method) && ! (contextPath + "/login").equals(requestURI)) {
response.sendRedirect(contextPath + "/login"); // redirect to the login page
return;
} else {
response.setStatus(HttpStatus.UNAUTHORIZED.value());
response.getWriter().write("Authentication required. Please provide a X-API-KEY in request header.\nThis is found in Settings -> Account Settings -> API Key\nAlternativly you can disable authentication if this is unexpected");
return;
}
}
}
filterChain.doFilter(request, response);
}
@@ -90,15 +92,17 @@ public class UserAuthenticationFilter extends OncePerRequestFilter {
@Override
protected boolean shouldNotFilter(HttpServletRequest request) throws ServletException {
String uri = request.getRequestURI();
String contextPath = request.getContextPath();
String[] permitAllPatterns = {
"/login",
"/register",
"/error",
"/images/",
"/public/",
"/css/",
"/js/"
contextPath + "/login",
contextPath + "/register",
contextPath + "/error",
contextPath + "/images/",
contextPath + "/public/",
contextPath + "/css/",
contextPath + "/js/",
contextPath + "/pdfjs/",
contextPath + "/site.webmanifest"
};
for (String pattern : permitAllPatterns) {

9
src/main/java/stirling/software/SPDF/config/security/UserService.java
View File

@@ -16,11 +16,13 @@ import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import stirling.software.SPDF.controller.api.pipeline.UserServiceInterface;
import stirling.software.SPDF.model.Authority;
import stirling.software.SPDF.model.Role;
import stirling.software.SPDF.model.User;
import stirling.software.SPDF.repository.UserRepository;
@Service
public class UserService {
public class UserService implements UserServiceInterface{
@Autowired
private UserRepository userRepository;
@@ -136,6 +138,11 @@ public class UserService {
public void deleteUser(String username) {
Optional<User> userOpt = userRepository.findByUsername(username);
if (userOpt.isPresent()) {
for (Authority authority : userOpt.get().getAuthorities()) {
if (authority.getAuthority().equals(Role.INTERNAL_API_USER.getRoleId())) {
return;
}
}
userRepository.delete(userOpt.get());
}
}

4
src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java
View File

@@ -29,12 +29,12 @@ import stirling.software.SPDF.model.api.SplitPdfBySectionsRequest;
import stirling.software.SPDF.utils.WebResponseUtils;
@RestController
@RequestMapping("/api/v1/general")
@Tag(name = "Misc", description = "Miscellaneous APIs")
@Tag(name = "General", description = "General APIs")
public class SplitPdfBySectionsController {
@PostMapping(value = "/split-pdf-by-sections", consumes = "multipart/form-data")
@Operation(summary = "Split PDF pages into smaller sections", description = "Split each page of a PDF into smaller sections based on the user's choice (halves, thirds, quarters, etc.), both vertically and horizontally. Input: PDF, Split Parameters. Output: ZIP containing split documents.")
@Operation(summary = "Split PDF pages into smaller sections", description = "Split each page of a PDF into smaller sections based on the user's choice (halves, thirds, quarters, etc.), both vertically and horizontally. Input:PDF Output:ZIP-PDF Type:SISO")
public ResponseEntity<byte[]> splitPdf(@ModelAttribute SplitPdfBySectionsRequest request) throws Exception {
List<ByteArrayOutputStream> splitDocumentsBoas = new ArrayList<>();

4
src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySizeController.java
View File

@@ -26,13 +26,13 @@ import stirling.software.SPDF.utils.WebResponseUtils;
@RestController
@RequestMapping("/api/v1/general")
@Tag(name = "Misc", description = "Miscellaneous APIs")
@Tag(name = "General", description = "General APIs")
public class SplitPdfBySizeController {
@PostMapping(value = "/split-by-size-or-count", consumes = "multipart/form-data")
@Operation(summary = "Auto split PDF pages into separate documents based on size or count", description = "split PDF into multiple paged documents based on size/count, ie if 20 pages and split into 5, it does 5 documents each 4 pages\r\n"
+ " if 10MB and each page is 1MB and you enter 2MB then 5 docs each 2MB (rounded so that it accepts 1.9MB but not 2.1MB) Input:PDF Output:ZIP Type:SIMO")
+ " if 10MB and each page is 1MB and you enter 2MB then 5 docs each 2MB (rounded so that it accepts 1.9MB but not 2.1MB) Input:PDF Output:ZIP-PDF Type:SISO")
public ResponseEntity<byte[]> autoSplitPdf(@ModelAttribute SplitPdfBySizeOrCountRequest request) throws Exception {
List<ByteArrayOutputStream> splitDocumentsBoas = new ArrayList<ByteArrayOutputStream>();

24
src/main/java/stirling/software/SPDF/controller/api/UserController.java
View File

@@ -23,6 +23,7 @@ import org.springframework.web.servlet.view.RedirectView;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import stirling.software.SPDF.config.security.UserService;
import stirling.software.SPDF.model.Role;
import stirling.software.SPDF.model.User;
@Controller
@@ -32,6 +33,7 @@ public class UserController {
@Autowired
private UserService userService;
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/register")
public String register(@RequestParam String username, @RequestParam String password, Model model) {
if(userService.usernameExists(username)) {
@@ -43,6 +45,7 @@ public class UserController {
return "redirect:/login?registered=true";
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/change-username-and-password")
public RedirectView changeUsernameAndPassword(Principal principal,
@RequestParam String currentPassword,
@@ -85,7 +88,7 @@ public class UserController {
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/change-username")
public RedirectView changeUsername(Principal principal,
@RequestParam String currentPassword,
@@ -122,7 +125,8 @@ public class UserController {
return new RedirectView("/login?messageType=credsUpdated");
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/change-password")
public RedirectView changePassword(Principal principal,
@RequestParam String currentPassword,
@@ -154,7 +158,7 @@ public class UserController {
return new RedirectView("/login?messageType=credsUpdated");
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/updateUserSettings")
public String updateUserSettings(HttpServletRequest request, Principal principal) {
Map<String, String[]> paramMap = request.getParameterMap();
@@ -182,6 +186,18 @@ public class UserController {
if(userService.usernameExists(username)) {
return new RedirectView("/addUsers?messageType=usernameExists");
}
try {
// Validate the role
Role roleEnum = Role.fromString(role);
if (roleEnum == Role.INTERNAL_API_USER) {
// If the role is INTERNAL_API_USER, reject the request
return new RedirectView("/addUsers?messageType=invalidRole");
}
} catch (IllegalArgumentException e) {
// If the role ID is not valid, redirect with an error message
return new RedirectView("/addUsers?messageType=invalidRole");
}
userService.saveUser(username, password, role, forceChange);
return new RedirectView("/addUsers"); // Redirect to account page after adding the user
}
@@ -203,6 +219,7 @@ public class UserController {
return "redirect:/addUsers";
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/get-api-key")
public ResponseEntity<String> getApiKey(Principal principal) {
if (principal == null) {
@@ -216,6 +233,7 @@ public class UserController {
return ResponseEntity.ok(apiKey);
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@PostMapping("/update-api-key")
public ResponseEntity<String> updateApiKey(Principal principal) {
if (principal == null) {

130
src/main/java/stirling/software/SPDF/controller/api/converters/ConvertEpubToPdf.java
View File

@@ -1,130 +0,0 @@
package stirling.software.SPDF.controller.api.converters;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import stirling.software.SPDF.model.api.GeneralFile;
import stirling.software.SPDF.utils.FileToPdf;
import stirling.software.SPDF.utils.WebResponseUtils;
@RestController
@RequestMapping("/api/v1/convert")
@Tag(name = "Convert", description = "Convert APIs")
public class ConvertEpubToPdf {
//TODO
@PostMapping(consumes = "multipart/form-data", value = "/epub-to-single-pdf")
@Hidden
@Operation(
summary = "Convert an EPUB file to a single PDF",
description = "This endpoint takes an EPUB file input and converts it to a single PDF."
)
public ResponseEntity<byte[]> epubToSinglePdf(
@ModelAttribute GeneralFile request)
throws Exception {
MultipartFile fileInput = request.getFileInput();
if (fileInput == null) {
throw new IllegalArgumentException("Please provide an EPUB file for conversion.");
}
String originalFilename = fileInput.getOriginalFilename();
if (originalFilename == null || !originalFilename.endsWith(".epub")) {
throw new IllegalArgumentException("File must be in .epub format.");
}
Map<String, byte[]> epubContents = extractEpubContent(fileInput);
List<String> htmlFilesOrder = getHtmlFilesOrderFromOpf(epubContents);
List<byte[]> individualPdfs = new ArrayList<>();
for (String htmlFile : htmlFilesOrder) {
byte[] htmlContent = epubContents.get(htmlFile);
byte[] pdfBytes = FileToPdf.convertHtmlToPdf(htmlContent, htmlFile.replace(".html", ".pdf"));
individualPdfs.add(pdfBytes);
}
// Pseudo-code to merge individual PDFs into one.
byte[] mergedPdfBytes = mergeMultiplePdfsIntoOne(individualPdfs);
return WebResponseUtils.bytesToWebResponse(mergedPdfBytes, originalFilename.replace(".epub", ".pdf"));
}
// Assuming a pseudo-code function that merges multiple PDFs into one.
private byte[] mergeMultiplePdfsIntoOne(List<byte[]> individualPdfs) {
// You can use a library such as PDFBox to perform the merging here.
// Return the byte[] of the merged PDF.
return null;
}
private Map<String, byte[]> extractEpubContent(MultipartFile fileInput) throws IOException {
Map<String, byte[]> contentMap = new HashMap<>();
try (ZipInputStream zis = new ZipInputStream(fileInput.getInputStream())) {
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int read = 0;
while ((read = zis.read(buffer)) != -1) {
baos.write(buffer, 0, read);
}
contentMap.put(zipEntry.getName(), baos.toByteArray());
zipEntry = zis.getNextEntry();
}
}
return contentMap;
}
private List<String> getHtmlFilesOrderFromOpf(Map<String, byte[]> epubContents) throws Exception {
String opfContent = new String(epubContents.get("OEBPS/content.opf")); // Adjusting for given path
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
InputSource is = new InputSource(new StringReader(opfContent));
Document doc = dBuilder.parse(is);
NodeList itemRefs = doc.getElementsByTagName("itemref");
List<String> htmlFilesOrder = new ArrayList<>();
for (int i = 0; i < itemRefs.getLength(); i++) {
Element itemRef = (Element) itemRefs.item(i);
String idref = itemRef.getAttribute("idref");
NodeList items = doc.getElementsByTagName("item");
for (int j = 0; j < items.getLength(); j++) {
Element item = (Element) items.item(j);
if (idref.equals(item.getAttribute("id"))) {
htmlFilesOrder.add(item.getAttribute("href")); // Fetching the actual href
break;
}
}
}
return htmlFilesOrder;
}
}

2
src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java
View File

@@ -28,7 +28,7 @@ public class ConvertWebsiteToPDF {
@PostMapping(consumes = "multipart/form-data", value = "/url/pdf")
@Operation(
summary = "Convert a URL to a PDF",
description = "This endpoint fetches content from a URL and converts it to a PDF format."
description = "This endpoint fetches content from a URL and converts it to a PDF format. Input:N/A Output:PDF Type:SISO"
)
public ResponseEntity<byte[]> urlToPdf(@ModelAttribute UrlToPdfRequest request) throws IOException, InterruptedException {
String URL = request.getUrlInput();

4
src/main/java/stirling/software/SPDF/controller/api/converters/ExtractController.java
View File

@@ -28,12 +28,12 @@ import stirling.software.SPDF.model.api.extract.PDFFilePage;
@RestController
@RequestMapping("/api/v1/convert")
@Tag(name = "General", description = "General APIs")
@Tag(name = "Convert", description = "Convert APIs")
public class ExtractController {
private static final Logger logger = LoggerFactory.getLogger(CropController.class);
@PostMapping(value = "/pdf-to-csv", consumes = "multipart/form-data")
@PostMapping(value = "/pdf/csv", consumes = "multipart/form-data")
@Operation(summary = "Extracts a PDF document to csv", description = "This operation takes an input PDF file and returns CSV file of whole page. Input:PDF Output:CSV Type:SISO")
public ResponseEntity<String> PdfToCsv(@ModelAttribute PDFFilePage form)
throws Exception {

2
src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java
View File

@@ -43,7 +43,7 @@ public class AutoSplitPdfController {
private static final String QR_CONTENT = "https://github.com/Frooodle/Stirling-PDF";
@PostMapping(value = "/auto-split-pdf", consumes = "multipart/form-data")
@Operation(summary = "Auto split PDF pages into separate documents", description = "This endpoint accepts a PDF file, scans each page for a specific QR code, and splits the document at the QR code boundaries. The output is a zip file containing each separate PDF document. Input:PDF Output:ZIP Type:SISO")
@Operation(summary = "Auto split PDF pages into separate documents", description = "This endpoint accepts a PDF file, scans each page for a specific QR code, and splits the document at the QR code boundaries. The output is a zip file containing each separate PDF document. Input:PDF Output:ZIP-PDF Type:SISO")
public ResponseEntity<byte[]> autoSplitPdf(@ModelAttribute AutoSplitPdfRequest request) throws IOException {
MultipartFile file = request.getFileInput();
boolean duplexMode = request.isDuplexMode();

2
src/main/java/stirling/software/SPDF/controller/api/misc/ShowJavascript.java
View File

@@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import stirling.software.SPDF.model.api.PDFFile;
import stirling.software.SPDF.utils.WebResponseUtils;
@@ -25,6 +26,7 @@ public class ShowJavascript {
private static final Logger logger = LoggerFactory.getLogger(ShowJavascript.class);
@PostMapping(consumes = "multipart/form-data", value = "/show-javascript")
@Operation(summary = "Grabs all JS from a PDF and returns a single JS file with all code", description = "desc. Input:PDF Output:JS Type:SISO")
public ResponseEntity<byte[]> extractHeader(@ModelAttribute PDFFile request) throws Exception {
MultipartFile inputFile = request.getFileInput();
String script = "";

119
src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java Normal file
View File

@@ -0,0 +1,119 @@
package stirling.software.SPDF.controller.api.pipeline;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.ServletContext;
import stirling.software.SPDF.model.ApiEndpoint;
import stirling.software.SPDF.model.Role;
@Service
public class ApiDocService {
private final Map<String, ApiEndpoint> apiDocumentation = new HashMap<>();
@Autowired
private ServletContext servletContext;
private String getApiDocsUrl() {
String contextPath = servletContext.getContextPath();
String port = System.getProperty("local.server.port");
if(port == null || port.length() == 0) {
port="8080";
}
return "http://localhost:"+ port + contextPath + "/v1/api-docs";
}
@Autowired(required=false)
private UserServiceInterface userService;
private String getApiKeyForUser() {
if(userService == null)
return "";
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
}
JsonNode apiDocsJsonRootNode;
//@EventListener(ApplicationReadyEvent.class)
private synchronized void loadApiDocumentation() {
try {
HttpHeaders headers = new HttpHeaders();
String apiKey = getApiKeyForUser();
if (!apiKey.isEmpty()) {
headers.set("X-API-KEY", apiKey);
}
HttpEntity<String> entity = new HttpEntity<>(headers);
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> response = restTemplate.exchange(getApiDocsUrl(), HttpMethod.GET, entity, String.class);
String apiDocsJson = response.getBody();
ObjectMapper mapper = new ObjectMapper();
apiDocsJsonRootNode = mapper.readTree(apiDocsJson);
JsonNode paths = apiDocsJsonRootNode.path("paths");
paths.fields().forEachRemaining(entry -> {
String path = entry.getKey();
JsonNode pathNode = entry.getValue();
if (pathNode.has("post")) {
JsonNode postNode = pathNode.get("post");
ApiEndpoint endpoint = new ApiEndpoint(path, postNode);
apiDocumentation.put(path, endpoint);
}
});
} catch (Exception e) {
// Handle exceptions
e.printStackTrace();
}
}
public boolean isValidOperation(String operationName, Map<String, Object> parameters) {
if(apiDocumentation.size() == 0) {
loadApiDocumentation();
}
if (!apiDocumentation.containsKey(operationName)) {
return false;
}
ApiEndpoint endpoint = apiDocumentation.get(operationName);
return endpoint.areParametersValid(parameters);
}
public boolean isMultiInput(String operationName) {
if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) {
loadApiDocumentation();
}
if (!apiDocumentation.containsKey(operationName)) {
return false;
}
ApiEndpoint endpoint = apiDocumentation.get(operationName);
String description = endpoint.getDescription();
Pattern pattern = Pattern.compile("Type:(\\w+)");
Matcher matcher = pattern.matcher(description);
if (matcher.find()) {
String type = matcher.group(1);
return type.startsWith("MI");
}
return false;
}
}
// Model class for API Endpoint

453
src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java
View File

@@ -1,56 +1,31 @@
package stirling.software.SPDF.controller.api.pipeline;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.v3.oas.annotations.tags.Tag;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.PipelineConfig;
import stirling.software.SPDF.model.PipelineOperation;
import stirling.software.SPDF.model.api.HandleDataRequest;
import stirling.software.SPDF.utils.WebResponseUtils;
@@ -60,373 +35,39 @@ import stirling.software.SPDF.utils.WebResponseUtils;
public class PipelineController {
private static final Logger logger = LoggerFactory.getLogger(PipelineController.class);
@Autowired
private ObjectMapper objectMapper;
final String jsonFileName = "pipelineConfig.json";
final String watchedFoldersDir = "./pipeline/watchedFolders/";
final String finishedFoldersDir = "./pipeline/finishedFolders/";
@Autowired
PipelineProcessor processor;
@Scheduled(fixedRate = 25000)
public void scanFolders() {
logger.info("Scanning folders...");
Path watchedFolderPath = Paths.get(watchedFoldersDir);
if (!Files.exists(watchedFolderPath)) {
try {
Files.createDirectories(watchedFolderPath);
logger.info("Created directory: {}", watchedFolderPath);
} catch (IOException e) {
logger.error("Error creating directory: {}", watchedFolderPath, e);
return;
}
}
try (Stream<Path> paths = Files.walk(watchedFolderPath)) {
paths.filter(Files::isDirectory).forEach(t -> {
try {
if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) {
handleDirectory(t);
}
} catch (Exception e) {
logger.error("Error handling directory: {}", t, e);
}
});
} catch (Exception e) {
logger.error("Error walking through directory: {}", watchedFolderPath, e);
}
}
@Autowired
ApplicationProperties applicationProperties;
private void handleDirectory(Path dir) throws Exception {
logger.info("Handling directory: {}", dir);
Path jsonFile = dir.resolve(jsonFileName);
Path processingDir = dir.resolve("processing"); // Directory to move files during processing
if (!Files.exists(processingDir)) {
Files.createDirectory(processingDir);
logger.info("Created processing directory: {}", processingDir);
}
if (Files.exists(jsonFile)) {
// Read JSON file
String jsonString;
try {
jsonString = new String(Files.readAllBytes(jsonFile));
logger.info("Read JSON file: {}", jsonFile);
} catch (IOException e) {
logger.error("Error reading JSON file: {}", jsonFile, e);
return;
}
// Decode JSON to PipelineConfig
PipelineConfig config;
try {
config = objectMapper.readValue(jsonString, PipelineConfig.class);
// Assuming your PipelineConfig class has getters for all necessary fields, you
// can perform checks here
if (config.getOperations() == null || config.getOutputDir() == null || config.getName() == null) {
throw new IOException("Invalid JSON format");
}
} catch (IOException e) {
logger.error("Error parsing PipelineConfig: {}", jsonString, e);
return;
}
// For each operation in the pipeline
for (PipelineOperation operation : config.getOperations()) {
// Collect all files based on fileInput
File[] files;
String fileInput = (String) operation.getParameters().get("fileInput");
if ("automated".equals(fileInput)) {
// If fileInput is "automated", process all files in the directory
try (Stream<Path> paths = Files.list(dir)) {
files = paths
.filter(path -> !Files.isDirectory(path)) // exclude directories
.filter(path -> !path.equals(jsonFile)) // exclude jsonFile
.map(Path::toFile)
.toArray(File[]::new);
} catch (IOException e) {
e.printStackTrace();
return;
}
} else {
// If fileInput contains a path, process only this file
files = new File[] { new File(fileInput) };
}
// Prepare the files for processing
List<File> filesToProcess = new ArrayList<>();
for (File file : files) {
logger.info(file.getName());
logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName()));
Files.move(file.toPath(), processingDir.resolve(file.getName()));
filesToProcess.add(processingDir.resolve(file.getName()).toFile());
}
// Process the files
try {
List<Resource> resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString);
if(resources == null) {
return;
}
// Move resultant files and rename them as per config in JSON file
for (Resource resource : resources) {
String resourceName = resource.getFilename();
String baseName = resourceName.substring(0, resourceName.lastIndexOf("."));
String extension = resourceName.substring(resourceName.lastIndexOf(".")+1);
String outputFileName = config.getOutputPattern().replace("{filename}", baseName);
outputFileName = outputFileName.replace("{pipelineName}", config.getName());
DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");
outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter));
DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss");
outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter));
outputFileName += "." + extension;
// {filename} {folder} {date} {tmime} {pipeline}
String outputDir = config.getOutputDir();
String outputFolder = applicationProperties.getAutoPipeline().getOutputFolder();
if (outputFolder == null || outputFolder.isEmpty()) {
// If the environment variable is not set, use the default value
outputFolder = finishedFoldersDir;
}
logger.info("outputDir 0={}", outputDir);
// Replace the placeholders in the outputDir string
outputDir = outputDir.replace("{outputFolder}", outputFolder);
outputDir = outputDir.replace("{folderName}", dir.toString());
logger.info("outputDir 1={}", outputDir);
outputDir = outputDir.replace("\\watchedFolders", "");
outputDir = outputDir.replace("//watchedFolders", "");
outputDir = outputDir.replace("\\\\watchedFolders", "");
outputDir = outputDir.replace("/watchedFolders", "");
Path outputPath;
logger.info("outputDir 2={}", outputDir);
if (Paths.get(outputDir).isAbsolute()) {
// If it's an absolute path, use it directly
outputPath = Paths.get(outputDir);
} else {
// If it's a relative path, make it relative to the current working directory
outputPath = Paths.get(".", outputDir);
}
logger.info("outputPath={}", outputPath);
if (!Files.exists(outputPath)) {
try {
Files.createDirectories(outputPath);
logger.info("Created directory: {}", outputPath);
} catch (IOException e) {
logger.error("Error creating directory: {}", outputPath, e);
return;
}
}
logger.info("outputPath {}", outputPath);
logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString());
File newFile = new File(outputPath.resolve(outputFileName).toString());
OutputStream os = new FileOutputStream(newFile);
os.write(((ByteArrayResource)resource).getByteArray());
os.close();
logger.info("made {}", outputPath.resolve(outputFileName));
}
// If successful, delete the original files
for (File file : filesToProcess) {
Files.deleteIfExists(processingDir.resolve(file.getName()));
}
} catch (Exception e) {
// If an error occurs, move the original files back
for (File file : filesToProcess) {
Files.move(processingDir.resolve(file.getName()), file.toPath());
}
throw e;
}
}
}
}
List<Resource> processFiles(List<Resource> outputFiles, String jsonString) throws Exception {
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(jsonString);
JsonNode pipelineNode = jsonNode.get("pipeline");
logger.info("Running pipelineNode: {}", pipelineNode);
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
PrintStream logPrintStream = new PrintStream(logStream);
boolean hasErrors = false;
for (JsonNode operationNode : pipelineNode) {
String operation = operationNode.get("operation").asText();
logger.info("Running operation: {}", operation);
JsonNode parametersNode = operationNode.get("parameters");
String inputFileExtension = "";
if (operationNode.has("inputFileType")) {
inputFileExtension = operationNode.get("inputFileType").asText();
} else {
inputFileExtension = ".pdf";
}
List<Resource> newOutputFiles = new ArrayList<>();
boolean hasInputFileType = false;
for (Resource file : outputFiles) {
if (file.getFilename().endsWith(inputFileExtension)) {
hasInputFileType = true;
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
body.add("fileInput", file);
Iterator<Map.Entry<String, JsonNode>> parameters = parametersNode.fields();
while (parameters.hasNext()) {
Map.Entry<String, JsonNode> parameter = parameters.next();
body.add(parameter.getKey(), parameter.getValue().asText());
}
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
RestTemplate restTemplate = new RestTemplate();
String url = "http://localhost:8080/" + operation;
ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
// If the operation is filter and the response body is null or empty, skip this file
if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) {
logger.info("Skipping file due to failing {}", operation);
continue;
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
logPrintStream.println("Error: " + response.getBody());
hasErrors = true;
continue;
}
// Define filename
String filename;
if ("auto-rename".equals(operation)) {
// If the operation is "auto-rename", generate a new filename.
// This is a simple example of generating a filename using current timestamp.
// Modify as per your needs.
filename = "file_" + System.currentTimeMillis();
} else {
// Otherwise, keep the original filename.
filename = file.getFilename();
}
// Check if the response body is a zip file
if (isZip(response.getBody())) {
// Unzip the file and add all the files to the new output files
newOutputFiles.addAll(unzip(response.getBody()));
} else {
Resource outputResource = new ByteArrayResource(response.getBody()) {
@Override
public String getFilename() {
return filename;
}
};
newOutputFiles.add(outputResource);
}
}
if (!hasInputFileType) {
logPrintStream.println(
"No files with extension " + inputFileExtension + " found for operation " + operation);
hasErrors = true;
}
outputFiles = newOutputFiles;
}
logPrintStream.close();
}
if (hasErrors) {
logger.error("Errors occurred during processing. Log: {}", logStream.toString());
}
return outputFiles;
}
List<Resource> handleFiles(File[] files, String jsonString) throws Exception {
if(files == null || files.length == 0) {
logger.info("No files");
return null;
}
logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(jsonString);
JsonNode pipelineNode = jsonNode.get("pipeline");
boolean hasErrors = false;
List<Resource> outputFiles = new ArrayList<>();
for (File file : files) {
Path path = Paths.get(file.getAbsolutePath());
System.out.println("Reading file: " + path); // debug statement
if (Files.exists(path)) {
Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) {
@Override
public String getFilename() {
return file.getName();
}
};
outputFiles.add(fileResource);
} else {
System.out.println("File not found: " + path); // debug statement
}
}
logger.info("Files successfully loaded. Starting processing...");
return processFiles(outputFiles, jsonString);
}
List<Resource> handleFiles(MultipartFile[] files, String jsonString) throws Exception {
if(files == null || files.length == 0) {
logger.info("No files");
return null;
}
logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length());
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(jsonString);
JsonNode pipelineNode = jsonNode.get("pipeline");
boolean hasErrors = false;
List<Resource> outputFiles = new ArrayList<>();
for (MultipartFile file : files) {
Resource fileResource = new ByteArrayResource(file.getBytes()) {
@Override
public String getFilename() {
return file.getOriginalFilename();
}
};
outputFiles.add(fileResource);
}
logger.info("Files successfully loaded. Starting processing...");
return processFiles(outputFiles, jsonString);
}
@Autowired
private ObjectMapper objectMapper;
@PostMapping("/handleData")
public ResponseEntity<byte[]> handleData(@ModelAttribute HandleDataRequest request) {
MultipartFile[] files = request.getFileInputs();
String jsonString = request.getJsonString();
public ResponseEntity<byte[]> handleData(@ModelAttribute HandleDataRequest request) throws JsonMappingException, JsonProcessingException {
if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
return new ResponseEntity<>(HttpStatus.BAD_REQUEST);
}
MultipartFile[] files = request.getFileInput();
String jsonString = request.getJson();
if (files == null) {
return null;
}
PipelineConfig config = objectMapper.readValue(jsonString, PipelineConfig.class);
logger.info("Received POST request to /handleData with {} files", files.length);
try {
List<Resource> outputFiles = handleFiles(files, jsonString);
List<Resource> inputFiles = processor.generateInputFiles(files);
if(inputFiles == null || inputFiles.size() == 0) {
return null;
}
List<Resource> outputFiles = processor.runPipelineAgainstFiles(inputFiles, config);
if (outputFiles != null && outputFiles.size() == 1) {
// If there is only one file, return it directly
Resource singleFile = outputFiles.get(0);
@@ -473,52 +114,4 @@ public class PipelineController {
}
}
private boolean isZip(byte[] data) {
if (data == null || data.length < 4) {
return false;
}
// Check the first four bytes of the data against the standard zip magic number
return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04;
}
private List<Resource> unzip(byte[] data) throws IOException {
logger.info("Unzipping data of length: {}", data.length);
List<Resource> unzippedFiles = new ArrayList<>();
try (ByteArrayInputStream bais = new ByteArrayInputStream(data);
ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int count;
while ((count = zis.read(buffer)) != -1) {
baos.write(buffer, 0, count);
}
final String filename = entry.getName();
Resource fileResource = new ByteArrayResource(baos.toByteArray()) {
@Override
public String getFilename() {
return filename;
}
};
// If the unzipped file is a zip file, unzip it
if (isZip(baos.toByteArray())) {
logger.info("File {} is a zip file. Unzipping...", filename);
unzippedFiles.addAll(unzip(baos.toByteArray()));
} else {
unzippedFiles.add(fileResource);
}
}
}
logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size());
return unzippedFiles;
}
}

258
src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java Normal file
View File

@@ -0,0 +1,258 @@
package stirling.software.SPDF.controller.api.pipeline;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import stirling.software.SPDF.model.ApplicationProperties;
import stirling.software.SPDF.model.PipelineConfig;
import stirling.software.SPDF.model.PipelineOperation;
@Service
public class PipelineDirectoryProcessor {
private static final Logger logger = LoggerFactory.getLogger(PipelineDirectoryProcessor.class);
@Autowired
private ObjectMapper objectMapper;
@Autowired
private ApiDocService apiDocService;
@Autowired
private ApplicationProperties applicationProperties;
final String watchedFoldersDir = "./pipeline/watchedFolders/";
final String finishedFoldersDir = "./pipeline/finishedFolders/";
@Autowired
PipelineProcessor processor;
@Scheduled(fixedRate = 60000)
public void scanFolders() {
if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) {
return;
}
Path watchedFolderPath = Paths.get(watchedFoldersDir);
if (!Files.exists(watchedFolderPath)) {
try {
Files.createDirectories(watchedFolderPath);
logger.info("Created directory: {}", watchedFolderPath);
} catch (IOException e) {
logger.error("Error creating directory: {}", watchedFolderPath, e);
return;
}
}
try (Stream<Path> paths = Files.walk(watchedFolderPath)) {
paths.filter(Files::isDirectory).forEach(t -> {
try {
if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) {
handleDirectory(t);
}
} catch (Exception e) {
logger.error("Error handling directory: {}", t, e);
}
});
} catch (Exception e) {
logger.error("Error walking through directory: {}", watchedFolderPath, e);
}
}
public void handleDirectory(Path dir) throws IOException {
logger.info("Handling directory: {}", dir);
Path processingDir = createProcessingDirectory(dir);
Optional<Path> jsonFileOptional = findJsonFile(dir);
if (!jsonFileOptional.isPresent()) {
logger.warn("No .JSON settings file found. No processing will happen for dir {}.", dir);
return;
}
Path jsonFile = jsonFileOptional.get();
PipelineConfig config = readAndParseJson(jsonFile);
processPipelineOperations(dir, processingDir, jsonFile, config);
}
private Path createProcessingDirectory(Path dir) throws IOException {
Path processingDir = dir.resolve("processing");
if (!Files.exists(processingDir)) {
Files.createDirectory(processingDir);
logger.info("Created processing directory: {}", processingDir);
}
return processingDir;
}
private Optional<Path> findJsonFile(Path dir) throws IOException {
try (Stream<Path> paths = Files.list(dir)) {
return paths.filter(file -> file.toString().endsWith(".json")).findFirst();
}
}
private PipelineConfig readAndParseJson(Path jsonFile) throws IOException {
String jsonString = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8);
logger.debug("Reading JSON file: {}", jsonFile);
return objectMapper.readValue(jsonString, PipelineConfig.class);
}
private void processPipelineOperations(Path dir, Path processingDir, Path jsonFile, PipelineConfig config) throws IOException {
for (PipelineOperation operation : config.getOperations()) {
validateOperation(operation);
File[] files = collectFilesForProcessing(dir, jsonFile, operation);
if(files == null || files.length == 0) {
logger.debug("No files detected for {} ", dir);
return;
}
List<File> filesToProcess = prepareFilesForProcessing(files, processingDir);
runPipelineAgainstFiles(filesToProcess, config, dir, processingDir);
}
}
private void validateOperation(PipelineOperation operation) throws IOException {
if (!apiDocService.isValidOperation(operation.getOperation(), operation.getParameters())) {
throw new IOException("Invalid operation: " + operation.getOperation());
}
}
private File[] collectFilesForProcessing(Path dir, Path jsonFile, PipelineOperation operation) throws IOException {
try (Stream<Path> paths = Files.list(dir)) {
if ("automated".equals(operation.getParameters().get("fileInput"))) {
return paths.filter(path -> !Files.isDirectory(path) && !path.equals(jsonFile))
.map(Path::toFile)
.toArray(File[]::new);
} else {
String fileInput = (String) operation.getParameters().get("fileInput");
return new File[]{new File(fileInput)};
}
}
}
private List<File> prepareFilesForProcessing(File[] files, Path processingDir) throws IOException {
List<File> filesToProcess = new ArrayList<>();
for (File file : files) {
Path targetPath = resolveUniqueFilePath(processingDir, file.getName());
Files.move(file.toPath(), targetPath);
filesToProcess.add(targetPath.toFile());
}
return filesToProcess;
}
private Path resolveUniqueFilePath(Path directory, String originalFileName) {
Path filePath = directory.resolve(originalFileName);
int counter = 1;
while (Files.exists(filePath)) {
String newName = appendSuffixToFileName(originalFileName, "(" + counter + ")");
filePath = directory.resolve(newName);
counter++;
}
return filePath;
}
private String appendSuffixToFileName(String originalFileName, String suffix) {
int dotIndex = originalFileName.lastIndexOf('.');
if (dotIndex == -1) {
return originalFileName + suffix;
} else {
return originalFileName.substring(0, dotIndex) + suffix + originalFileName.substring(dotIndex);
}
}
private void runPipelineAgainstFiles(List<File> filesToProcess, PipelineConfig config, Path dir, Path processingDir) throws IOException {
try {
List<Resource> inputFiles = processor.generateInputFiles(filesToProcess.toArray(new File[0]));
if(inputFiles == null || inputFiles.size() == 0) {
return;
}
List<Resource> outputFiles = processor.runPipelineAgainstFiles(inputFiles, config);
if (outputFiles == null) return;
moveAndRenameFiles(outputFiles, config, dir);
deleteOriginalFiles(filesToProcess, processingDir);
} catch (Exception e) {
logger.error("error during processing", e);
moveFilesBack(filesToProcess, processingDir);
}
}
private void moveAndRenameFiles(List<Resource> resources, PipelineConfig config, Path dir) throws IOException {
for (Resource resource : resources) {
String outputFileName = createOutputFileName(resource, config);
Path outputPath = determineOutputPath(config, dir);
if (!Files.exists(outputPath)) {
Files.createDirectories(outputPath);
logger.info("Created directory: {}", outputPath);
}
Path outputFile = outputPath.resolve(outputFileName);
try (OutputStream os = new FileOutputStream(outputFile.toFile())) {
os.write(((ByteArrayResource) resource).getByteArray());
}
logger.info("File moved and renamed to {}", outputFile);
}
}
private String createOutputFileName(Resource resource, PipelineConfig config) {
String resourceName = resource.getFilename();
String baseName = resourceName.substring(0, resourceName.lastIndexOf('.'));
String extension = resourceName.substring(resourceName.lastIndexOf('.') + 1);
String outputFileName = config.getOutputPattern()
.replace("{filename}", baseName)
.replace("{pipelineName}", config.getName())
.replace("{date}", LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")))
.replace("{time}", LocalTime.now().format(DateTimeFormatter.ofPattern("HHmmss")))
+ "." + extension;
return outputFileName;
}
private Path determineOutputPath(PipelineConfig config, Path dir) {
String outputDir = config.getOutputDir()
.replace("{outputFolder}", finishedFoldersDir)
.replace("{folderName}", dir.toString())
.replaceAll("\\\\?watchedFolders", "");
return Paths.get(outputDir).isAbsolute() ? Paths.get(outputDir) : Paths.get(".", outputDir);
}
private void deleteOriginalFiles(List<File> filesToProcess, Path processingDir) throws IOException {
for (File file : filesToProcess) {
Files.deleteIfExists(processingDir.resolve(file.getName()));
logger.info("Deleted original file: {}", file.getName());
}
}
private void moveFilesBack(List<File> filesToProcess, Path processingDir) {
for (File file : filesToProcess) {
try {
Files.move(processingDir.resolve(file.getName()), file.toPath());
logger.info("Moved file back to original location: {} , {}",file.toPath(), file.getName());
} catch (IOException e) {
logger.error("Error moving file back to original location: {}", file.getName(), e);
}
}
}
}

328
src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java Normal file
View File

@@ -0,0 +1,328 @@
package stirling.software.SPDF.controller.api.pipeline;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import jakarta.servlet.ServletContext;
import stirling.software.SPDF.model.PipelineConfig;
import stirling.software.SPDF.model.PipelineOperation;
import stirling.software.SPDF.model.Role;
@Service
public class PipelineProcessor {
private static final Logger logger = LoggerFactory.getLogger(PipelineProcessor.class);
@Autowired
private ApiDocService apiDocService;
@Autowired
private UserServiceInterface userService;
@Autowired
private ServletContext servletContext;
private String getApiKeyForUser() {
if (userService == null)
return "";
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
}
private String getBaseUrl() {
String contextPath = servletContext.getContextPath();
return "http://localhost:8080" + contextPath + "/";
}
List<Resource> runPipelineAgainstFiles(List<Resource> outputFiles, PipelineConfig config) throws Exception {
ByteArrayOutputStream logStream = new ByteArrayOutputStream();
PrintStream logPrintStream = new PrintStream(logStream);
boolean hasErrors = false;
for (PipelineOperation pipelineOperation : config.getOperations()) {
String operation = pipelineOperation.getOperation();
boolean isMultiInputOperation = apiDocService.isMultiInput(operation);
logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation);
Map<String, Object> parameters = pipelineOperation.getParameters();
String inputFileExtension = "";
//TODO
//if (operationNode.has("inputFileType")) {
// inputFileExtension = operationNode.get("inputFileType").asText();
//} else {
inputFileExtension = ".pdf";
//}
final String finalInputFileExtension = inputFileExtension;
String url = getBaseUrl() + operation;
List<Resource> newOutputFiles = new ArrayList<>();
if (!isMultiInputOperation) {
for (Resource file : outputFiles) {
boolean hasInputFileType = false;
if (file.getFilename().endsWith(inputFileExtension)) {
hasInputFileType = true;
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
body.add("fileInput", file);
for(Entry<String, Object> entry : parameters.entrySet()) {
body.add(entry.getKey(), entry.getValue());
}
ResponseEntity<byte[]> response = sendWebRequest(url, body);
// If the operation is filter and the response body is null or empty, skip this
// file
if (operation.startsWith("filter-")
&& (response.getBody() == null || response.getBody().length == 0)) {
logger.info("Skipping file due to failing {}", operation);
continue;
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
logPrintStream.println("Error: " + response.getBody());
hasErrors = true;
continue;
}
processOutputFiles(operation, file.getFilename(), response, newOutputFiles);
}
if (!hasInputFileType) {
logPrintStream.println(
"No files with extension " + inputFileExtension + " found for operation " + operation);
hasErrors = true;
}
outputFiles = newOutputFiles;
}
} else {
// Filter and collect all files that match the inputFileExtension
List<Resource> matchingFiles = outputFiles.stream()
.filter(file -> file.getFilename().endsWith(finalInputFileExtension))
.collect(Collectors.toList());
// Check if there are matching files
if (!matchingFiles.isEmpty()) {
// Create a new MultiValueMap for the request body
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
// Add all matching files to the body
for (Resource file : matchingFiles) {
body.add("fileInput", file);
}
for(Entry<String, Object> entry : parameters.entrySet()) {
body.add(entry.getKey(), entry.getValue());
}
ResponseEntity<byte[]> response = sendWebRequest(url, body);
// Handle the response
if (response.getStatusCode().equals(HttpStatus.OK)) {
processOutputFiles(operation, matchingFiles.get(0).getFilename(), response, newOutputFiles);
} else {
// Log error if the response status is not OK
logPrintStream.println("Error in multi-input operation: " + response.getBody());
hasErrors = true;
}
} else {
logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation);
hasErrors = true;
}
}
logPrintStream.close();
}
if (hasErrors) {
logger.error("Errors occurred during processing. Log: {}", logStream.toString());
}
return outputFiles;
}
private ResponseEntity<byte[]> sendWebRequest(String url, MultiValueMap<String, Object> body ){
RestTemplate restTemplate = new RestTemplate();
// Set up headers, including API key
HttpHeaders headers = new HttpHeaders();
String apiKey = getApiKeyForUser();
headers.add("X-API-Key", apiKey);
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
// Create HttpEntity with the body and headers
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
// Make the request to the REST endpoint
return restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class);
}
private List<Resource> processOutputFiles(String operation, String fileName, ResponseEntity<byte[]> response, List<Resource> newOutputFiles) throws IOException{
// Define filename
String newFilename;
if ("auto-rename".equals(operation)) {
// If the operation is "auto-rename", generate a new filename.
// This is a simple example of generating a filename using current timestamp.
// Modify as per your needs.
newFilename = "file_" + System.currentTimeMillis();
} else {
// Otherwise, keep the original filename.
newFilename = fileName;
}
// Check if the response body is a zip file
if (isZip(response.getBody())) {
// Unzip the file and add all the files to the new output files
newOutputFiles.addAll(unzip(response.getBody()));
} else {
Resource outputResource = new ByteArrayResource(response.getBody()) {
@Override
public String getFilename() {
return newFilename;
}
};
newOutputFiles.add(outputResource);
}
return newOutputFiles;
}
List<Resource> generateInputFiles(File[] files) throws Exception {
if (files == null || files.length == 0) {
logger.info("No files");
return null;
}
List<Resource> outputFiles = new ArrayList<>();
for (File file : files) {
Path path = Paths.get(file.getAbsolutePath());
logger.info("Reading file: " + path); // debug statement
if (Files.exists(path)) {
Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) {
@Override
public String getFilename() {
return file.getName();
}
};
outputFiles.add(fileResource);
} else {
logger.info("File not found: " + path);
}
}
logger.info("Files successfully loaded. Starting processing...");
return outputFiles;
}
List<Resource> generateInputFiles(MultipartFile[] files) throws Exception {
if (files == null || files.length == 0) {
logger.info("No files");
return null;
}
List<Resource> outputFiles = new ArrayList<>();
for (MultipartFile file : files) {
Resource fileResource = new ByteArrayResource(file.getBytes()) {
@Override
public String getFilename() {
return file.getOriginalFilename();
}
};
outputFiles.add(fileResource);
}
logger.info("Files successfully loaded. Starting processing...");
return outputFiles;
}
private boolean isZip(byte[] data) {
if (data == null || data.length < 4) {
return false;
}
// Check the first four bytes of the data against the standard zip magic number
return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04;
}
private List<Resource> unzip(byte[] data) throws IOException {
logger.info("Unzipping data of length: {}", data.length);
List<Resource> unzippedFiles = new ArrayList<>();
try (ByteArrayInputStream bais = new ByteArrayInputStream(data);
ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int count;
while ((count = zis.read(buffer)) != -1) {
baos.write(buffer, 0, count);
}
final String filename = entry.getName();
Resource fileResource = new ByteArrayResource(baos.toByteArray()) {
@Override
public String getFilename() {
return filename;
}
};
// If the unzipped file is a zip file, unzip it
if (isZip(baos.toByteArray())) {
logger.info("File {} is a zip file. Unzipping...", filename);
unzippedFiles.addAll(unzip(baos.toByteArray()));
} else {
unzippedFiles.add(fileResource);
}
}
}
logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size());
return unzippedFiles;
}
}

4
src/main/java/stirling/software/SPDF/controller/api/pipeline/UserServiceInterface.java Normal file
View File

@@ -0,0 +1,4 @@
package stirling.software.SPDF.controller.api.pipeline;
public interface UserServiceInterface {
String getApiKeyForUser(String username);
}

23
src/main/java/stirling/software/SPDF/controller/web/AccountWebController.java
View File

@@ -1,4 +1,5 @@
package stirling.software.SPDF.controller.web;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
@@ -15,6 +16,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import stirling.software.SPDF.model.Authority;
import stirling.software.SPDF.model.Role;
import stirling.software.SPDF.model.User;
import stirling.software.SPDF.repository.UserRepository;
@Controller
@@ -46,14 +49,28 @@ public class AccountWebController {
@PreAuthorize("hasRole('ROLE_ADMIN')")
@GetMapping("/addUsers")
public String showAddUserForm(Model model, Authentication authentication) {
List<User> allUsers = userRepository.findAll();
List<User> allUsers = userRepository.findAll();
Iterator<User> iterator = allUsers.iterator();
while(iterator.hasNext()) {
User user = iterator.next();
if(user != null) {
for (Authority authority : user.getAuthorities()) {
if (authority.getAuthority().equals(Role.INTERNAL_API_USER.getRoleId())) {
iterator.remove();
break; // Break out of the inner loop once the user is removed
}
}
}
}
model.addAttribute("users", allUsers);
model.addAttribute("currentUsername", authentication.getName());
return "addUsers";
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@GetMapping("/account")
public String account(HttpServletRequest request, Model model, Authentication authentication) {
if (authentication == null || !authentication.isAuthenticated()) {
@@ -100,7 +117,7 @@ public class AccountWebController {
}
@PreAuthorize("!hasAuthority('ROLE_DEMO_USER')")
@GetMapping("/change-creds")
public String changeCreds(HttpServletRequest request, Model model, Authentication authentication) {
if (authentication == null || !authentication.isAuthenticated()) {

60
src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java
View File

@@ -1,5 +1,6 @@
package stirling.software.SPDF.controller.web;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -41,31 +42,44 @@ public class GeneralWebController {
model.addAttribute("currentPage", "pipeline");
List<String> pipelineConfigs = new ArrayList<>();
try (Stream<Path> paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) {
List<Path> jsonFiles = paths
.filter(Files::isRegularFile)
.filter(p -> p.toString().endsWith(".json"))
.collect(Collectors.toList());
for (Path jsonFile : jsonFiles) {
String content = Files.readString(jsonFile, StandardCharsets.UTF_8);
pipelineConfigs.add(content);
}
List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
for (String config : pipelineConfigs) {
Map<String, Object> jsonContent = new ObjectMapper().readValue(config, new TypeReference<Map<String, Object>>(){});
String name = (String) jsonContent.get("name");
Map<String, String> configWithName = new HashMap<>();
configWithName.put("json", config);
configWithName.put("name", name);
List<Map<String, String>> pipelineConfigsWithNames = new ArrayList<>();
if(new File("./pipeline/defaultWebUIConfigs/").exists()) {
try (Stream<Path> paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) {
List<Path> jsonFiles = paths
.filter(Files::isRegularFile)
.filter(p -> p.toString().endsWith(".json"))
.collect(Collectors.toList());
for (Path jsonFile : jsonFiles) {
String content = Files.readString(jsonFile, StandardCharsets.UTF_8);
pipelineConfigs.add(content);
}
for (String config : pipelineConfigs) {
Map<String, Object> jsonContent = new ObjectMapper().readValue(config, new TypeReference<Map<String, Object>>(){});
String name = (String) jsonContent.get("name");
Map<String, String> configWithName = new HashMap<>();
configWithName.put("json", config);
configWithName.put("name", name);
pipelineConfigsWithNames.add(configWithName);
}
} catch (IOException e) {
e.printStackTrace();
}
}
if(pipelineConfigsWithNames.size() == 0) {
Map<String, String> configWithName = new HashMap<>();
configWithName.put("json", "");
configWithName.put("name", "No preloaded configs found");
pipelineConfigsWithNames.add(configWithName);
}
model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
} catch (IOException e) {
e.printStackTrace();
}
model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames);
model.addAttribute("pipelineConfigs", pipelineConfigs);

4
src/main/java/stirling/software/SPDF/controller/web/MetricsController.java
View File

@@ -27,8 +27,8 @@ import stirling.software.SPDF.config.StartupApplicationListener;
import stirling.software.SPDF.model.ApplicationProperties;
@RestController
@RequestMapping("/api/v1")
@Tag(name = "API", description = "Info APIs")
@RequestMapping("/api/v1/info")
@Tag(name = "Info", description = "Info APIs")
public class MetricsController {

42
src/main/java/stirling/software/SPDF/model/ApiEndpoint.java Normal file
View File

@@ -0,0 +1,42 @@
package stirling.software.SPDF.model;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.databind.JsonNode;
public class ApiEndpoint {
private String name;
private Map<String, JsonNode> parameters;
private String description;
public ApiEndpoint(String name, JsonNode postNode) {
this.name = name;
this.parameters = new HashMap<>();
postNode.path("parameters").forEach(paramNode -> {
String paramName = paramNode.path("name").asText();
parameters.put(paramName, paramNode);
});
this.description = postNode.path("description").asText();
}
public boolean areParametersValid(Map<String, Object> providedParams) {
for (String requiredParam : parameters.keySet()) {
if (!providedParams.containsKey(requiredParam)) {
return false;
}
}
return true;
}
public String getDescription() {
return description;
}
@Override
public String toString() {
return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]";
}
}

39
src/main/java/stirling/software/SPDF/model/ApplicationProperties.java
View File

@@ -106,6 +106,25 @@ public class ApplicationProperties {
private Boolean enableLogin;
private Boolean csrfDisabled;
private InitialLogin initialLogin;
private int loginAttemptCount;
private long loginResetTimeMinutes;
public int getLoginAttemptCount() {
return loginAttemptCount;
}
public void setLoginAttemptCount(int loginAttemptCount) {
this.loginAttemptCount = loginAttemptCount;
}
public long getLoginResetTimeMinutes() {
return loginResetTimeMinutes;
}
public void setLoginResetTimeMinutes(long loginResetTimeMinutes) {
this.loginResetTimeMinutes = loginResetTimeMinutes;
}
public InitialLogin getInitialLogin() {
return initialLogin != null ? initialLogin : new InitialLogin();
@@ -175,6 +194,19 @@ public class ApplicationProperties {
private String rootURIPath;
private String customStaticFilePath;
private Integer maxFileSize;
private Boolean enableAlphaFunctionality;
public Boolean getEnableAlphaFunctionality() {
return enableAlphaFunctionality;
}
public void setEnableAlphaFunctionality(Boolean enableAlphaFunctionality) {
this.enableAlphaFunctionality = enableAlphaFunctionality;
}
public String getDefaultLocale() {
return defaultLocale;
@@ -218,12 +250,13 @@ public class ApplicationProperties {
@Override
public String toString() {
return "System [defaultLocale=" + defaultLocale + ", googlevisibility=" + googlevisibility + ", rootURIPath="
+ rootURIPath + ", customStaticFilePath=" + customStaticFilePath + ", maxFileSize=" + maxFileSize
+ "]";
return "System [defaultLocale=" + defaultLocale + ", googlevisibility=" + googlevisibility
+ ", rootURIPath=" + rootURIPath + ", customStaticFilePath=" + customStaticFilePath
+ ", maxFileSize=" + maxFileSize + ", enableAlphaFunctionality=" + enableAlphaFunctionality + "]";
}
}
public static class Ui {

27
src/main/java/stirling/software/SPDF/model/AttemptCounter.java Normal file
View File

@@ -0,0 +1,27 @@
package stirling.software.SPDF.model;
public class AttemptCounter {
private int attemptCount;
private long lastAttemptTime;
public AttemptCounter() {
this.attemptCount = 1;
this.lastAttemptTime = System.currentTimeMillis();
}
public void increment() {
this.attemptCount++;
this.lastAttemptTime = System.currentTimeMillis();
}
public int getAttemptCount() {
return attemptCount;
}
public long getlastAttemptTime() {
return lastAttemptTime;
}
public boolean shouldReset(long ATTEMPT_INCREMENT_TIME) {
return System.currentTimeMillis() - lastAttemptTime > ATTEMPT_INCREMENT_TIME;
}
}

7
src/main/java/stirling/software/SPDF/model/Role.java
View File

@@ -14,8 +14,13 @@ public enum Role {
EXTRA_LIMITED_API_USER("ROLE_EXTRA_LIMITED_API_USER", 20, 20),
// 0 API calls per day and 20 web calls
WEB_ONLY_USER("ROLE_WEB_ONLY_USER", 0, 20);
WEB_ONLY_USER("ROLE_WEB_ONLY_USER", 0, 20),
INTERNAL_API_USER("STIRLING-PDF-BACKEND-API-USER", Integer.MAX_VALUE, Integer.MAX_VALUE),
DEMO_USER("ROLE_DEMO_USER", 100, 100);
private final String roleId;
private final int apiCallsPerDay;
private final int webCallsPerDay;

4
src/main/java/stirling/software/SPDF/model/api/HandleDataRequest.java
View File

@@ -13,8 +13,8 @@ import lombok.NoArgsConstructor;
public class HandleDataRequest {
@Schema(description = "The input files")
private MultipartFile[] fileInputs;
private MultipartFile[] fileInput;
@Schema(description = "JSON String")
private String jsonString;
private String json;
}

5
src/main/java/stirling/software/SPDF/model/api/PDFWithPageNums.java
View File

@@ -5,6 +5,7 @@ import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -18,7 +19,7 @@ public class PDFWithPageNums extends PDFFile {
@Schema(description = "The pages to select, Supports ranges (e.g., '1,3,5-9'), or 'all' or functions in the format 'an+b' where 'a' is the multiplier of the page number 'n', and 'b' is a constant (e.g., '2n+1', '3n', '6n-5')\"")
private String pageNumbers;
@Hidden
public List<Integer> getPageNumbersList(){
int pageCount = 0;
try {
@@ -30,6 +31,8 @@ public class PDFWithPageNums extends PDFFile {
return GeneralUtils.parsePageString(pageNumbers, pageCount);
}
@Hidden
public List<Integer> getPageNumbersList(PDDocument doc){
int pageCount = 0;
pageCount = doc.getNumberOfPages();

9
src/main/java/stirling/software/SPDF/utils/PdfUtils.java
View File

@@ -4,22 +4,17 @@ import java.awt.Graphics;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.imageio.ImageIO;
import javax.imageio.IIOImage;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.ImageWriter;
import javax.imageio.ImageWriteParam;
import javax.imageio.ImageWriter;
import javax.imageio.stream.ImageOutputStream;
import org.apache.pdfbox.pdmodel.PDDocument;

16
src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java Normal file
View File

@@ -0,0 +1,16 @@
package stirling.software.SPDF.utils;
public class RequestUriUtils {
public static boolean isStaticResource(String requestURI) {
return requestURI.startsWith("/css/")
|| requestURI.startsWith("/js/")
|| requestURI.startsWith("/images/")
|| requestURI.startsWith("/public/")
|| requestURI.startsWith("/pdfjs/")
|| requestURI.endsWith(".svg");
}
}