Skip to content

Commit

Permalink
Pre-fetch project information on startup
Browse files Browse the repository at this point in the history
This avoid hitting Github's rate limit as data is not fetched
on every request. A Github webhook triggers the cache update when
content changes in the spring-io/spring-website-content
  • Loading branch information
mbhave committed Nov 6, 2024
1 parent 8f57f4f commit 6b1c70c
Show file tree
Hide file tree
Showing 20 changed files with 606 additions and 77 deletions.
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-graphql'
implementation 'com.azure.spring:spring-cloud-azure-starter-keyvault-secrets'
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
implementation 'com.github.ben-manes.caffeine:caffeine'
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
implementation 'com.vladsch.flexmark:flexmark-all:0.64.8'
implementation 'org.apache.maven:maven-artifact:3.6.3'
testImplementation 'com.squareup.okhttp3:mockwebserver'
Expand Down
2 changes: 0 additions & 2 deletions src/main/java/io/spring/projectapi/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,9 @@
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;

@SpringBootApplication
@EnableCaching
@EnableConfigurationProperties(ApplicationProperties.class)
public class Application {

Expand Down
12 changes: 11 additions & 1 deletion src/main/java/io/spring/projectapi/ApplicationProperties.java
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,18 @@ public static class Github {
*/
private String branch;

/**
* Secret for triggering the webhook that refreshes the cache.
*/
private String webhookSecret;

@ConstructorBinding
Github(String org, String team, String accesstoken, @DefaultValue("main") String branch) {
Github(String org, String team, String accesstoken, @DefaultValue("main") String branch, String webhookSecret) {
this.org = org;
this.team = team;
this.accesstoken = accesstoken;
this.branch = branch;
this.webhookSecret = webhookSecret;
}

public String getOrg() {
Expand All @@ -92,6 +98,10 @@ public String getAccesstoken() {
return this.accesstoken;
}

public String getWebhookSecret() {
return this.webhookSecret;
}

}

}
96 changes: 96 additions & 0 deletions src/main/java/io/spring/projectapi/ProjectRepository.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* Copyright 2022-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.spring.projectapi;

import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import io.spring.projectapi.github.GithubOperations;
import io.spring.projectapi.github.Project;
import io.spring.projectapi.github.ProjectDocumentation;
import io.spring.projectapi.github.ProjectSupport;
import io.spring.projectapi.web.webhook.CacheController;

import org.springframework.stereotype.Component;

/**
* Caches Github project information. Populated on start up and updates triggered via
* {@link CacheController}.
*
* @author Madhura Bhave
* @author Phillip Webb
*/
@Component
public class ProjectRepository {

private final GithubOperations githubOperations;

private transient Data data;

public ProjectRepository(GithubOperations githubOperations) {
this.githubOperations = githubOperations;
this.data = Data.load(githubOperations);
}

public void update() {
this.data = Data.load(this.githubOperations);
}

public Collection<Project> getProjects() {
return this.data.project().values();
}

public Project getProject(String projectSlug) {
return this.data.project().get(projectSlug);
}

public List<ProjectDocumentation> getProjectDocumentations(String projectSlug) {
return this.data.documentation().get(projectSlug);
}

public List<ProjectSupport> getProjectSupports(String projectSlug) {
return this.data.support().get(projectSlug);
}

public String getProjectSupportPolicy(String projectSlug) {
return this.data.supportPolicy().get(projectSlug);
}

record Data(Map<String, Project> project, Map<String, List<ProjectDocumentation>> documentation,
Map<String, List<ProjectSupport>> support, Map<String, String> supportPolicy) {

public static Data load(GithubOperations githubOperations) {
Map<String, Project> projects = new LinkedHashMap<>();
Map<String, List<ProjectDocumentation>> documentation = new LinkedHashMap<>();
Map<String, List<ProjectSupport>> support = new LinkedHashMap<>();
Map<String, String> supportPolicy = new LinkedHashMap<>();
githubOperations.getProjects().forEach((project) -> {
String slug = project.getSlug();
projects.put(slug, project);
documentation.put(slug, githubOperations.getProjectDocumentations(slug));
support.put(slug, githubOperations.getProjectSupports(slug));
supportPolicy.put(slug, githubOperations.getProjectSupportPolicy(slug));
});
return new Data(Map.copyOf(projects), Map.copyOf(documentation), Map.copyOf(support),
Map.copyOf(supportPolicy));
}

}

}
41 changes: 15 additions & 26 deletions src/main/java/io/spring/projectapi/github/GithubOperations.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@
import io.spring.projectapi.github.ProjectDocumentation.Status;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.cache.annotation.Cacheable;
Expand All @@ -56,6 +54,12 @@
*/
public class GithubOperations {

private static final TypeReference<@NotNull List<ProjectDocumentation>> DOCUMENTATION_LIST = new TypeReference<>() {
};

private static final TypeReference<List<ProjectSupport>> SUPPORT_LIST = new TypeReference<>() {
};

private static final String GITHUB_URI = "https://api.github.com/repos/spring-io/spring-website-content/contents";

private static final Comparator<ProjectDocumentation> VERSION_COMPARATOR = GithubOperations::compare;
Expand All @@ -80,8 +84,6 @@ public class GithubOperations {

private final String branch;

private static final Logger logger = LoggerFactory.getLogger(GithubOperations.class);

public GithubOperations(RestTemplateBuilder restTemplateBuilder, ObjectMapper objectMapper, String token,
String branch) {
this.restTemplate = restTemplateBuilder.rootUri(GITHUB_URI)
Expand Down Expand Up @@ -115,13 +117,7 @@ public void addProjectDocumentation(String projectSlug, ProjectDocumentation doc

@NotNull
private List<ProjectDocumentation> convertToProjectDocumentation(String content) {
try {
return this.objectMapper.readValue(content, new TypeReference<>() {
});
}
catch (JsonProcessingException ex) {
throw new RuntimeException(ex);
}
return readValue(content, DOCUMENTATION_LIST);
}

private void updateProjectDocumentation(String projectSlug, List<ProjectDocumentation> documentations, String sha) {
Expand Down Expand Up @@ -180,7 +176,7 @@ private List<ProjectDocumentation> computeCurrentRelease(List<ProjectDocumentati
List<ProjectDocumentation> updatedGaList = new ArrayList<>(getListWithUpdatedCurrentRelease(sortedGaList));
Collections.reverse(updatedGaList);
preReleaseList.addAll(updatedGaList);
return preReleaseList;
return List.copyOf(preReleaseList);
}

@NotNull
Expand Down Expand Up @@ -209,15 +205,13 @@ public void deleteDocumentation(String projectSlug, String version) {
}

private ResponseEntity<Map<String, Object>> getFile(String projectSlug, String fileName) {
logger.info("****In private getFile for project " + projectSlug);
RequestEntity<Void> request = RequestEntity
.get("/project/{projectSlug}/{fileName}?ref=" + this.branch, projectSlug, fileName)
.build();
try {
return this.restTemplate.exchange(request, STRING_OBJECT_MAP);
}
catch (HttpClientErrorException ex) {
logger.info("****In private getFile for project with exception " + projectSlug);
HttpStatusCode statusCode = ex.getStatusCode();
if (statusCode.value() == 404) {
throwIfProjectDoesNotExist(projectSlug);
Expand Down Expand Up @@ -255,7 +249,6 @@ private String getFileSha(ResponseEntity<Map<String, Object>> exchange) {
public List<Project> getProjects() {
List<Project> projects = new ArrayList<>();
try {
logger.info("****In getProjects");
RequestEntity<Void> request = RequestEntity.get("/project?ref=" + this.branch).build();
ResponseEntity<List<Map<String, Object>>> exchange = this.restTemplate.exchange(request,
STRING_OBJECT_MAP_LIST);
Expand All @@ -274,12 +267,10 @@ public List<Project> getProjects() {
catch (HttpClientErrorException ex) {
// Return empty list
}
return projects;
return List.copyOf(projects);
}

@Cacheable(value = "project", key = "#projectSlug")
public Project getProject(String projectSlug) {
logger.info("****In getProject with project " + projectSlug);
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "index.md");
String contents = getFileContents(response);
Map<String, String> frontMatter = MarkdownUtils.getFrontMatter(contents);
Expand All @@ -288,26 +279,25 @@ public Project getProject(String projectSlug) {
return this.objectMapper.convertValue(frontMatter, Project.class);
}

@Cacheable(value = "documentation", key = "#projectSlug")
public List<ProjectDocumentation> getProjectDocumentations(String projectSlug) {
logger.info("****In getProjectDocumentations with project " + projectSlug);
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "documentation.json");
String content = getFileContents(response);
return convertToProjectDocumentation(content);
return List.copyOf(convertToProjectDocumentation(content));
}

@Cacheable(value = "support", key = "#projectSlug")
public List<ProjectSupport> getProjectSupports(String projectSlug) {
logger.info("****In getProjectSupports with project " + projectSlug);
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "support.json");
if (response == null) {
return Collections.emptyList();
}
String contents = getFileContents(response);
getProjectSupportPolicy(projectSlug);
return List.copyOf(readValue(contents, SUPPORT_LIST));
}

private <T> T readValue(String contents, TypeReference<T> type) {
try {
return this.objectMapper.readValue(contents, new TypeReference<>() {
});
return this.objectMapper.readValue(contents, type);
}
catch (JsonProcessingException ex) {
throw new RuntimeException(ex);
Expand All @@ -316,7 +306,6 @@ public List<ProjectSupport> getProjectSupports(String projectSlug) {

@Cacheable(value = "support_policy", key = "#projectSlug")
public String getProjectSupportPolicy(String projectSlug) {
logger.info("****In getProjectSupportPolicy with project " + projectSlug);
ResponseEntity<Map<String, Object>> indexResponse = getFile(projectSlug, "index.md");
String indexContents = getFileContents(indexResponse);
Map<String, String> frontMatter = MarkdownUtils.getFrontMatter(indexContents);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ public SecurityFilterChain configure(HttpSecurity http, RestTemplateBuilder rest
http.requiresChannel((channel) -> channel.requestMatchers(this::hasXForwardedPortHeader).requiresSecure());
http.authorizeHttpRequests((requests) -> {
requests.requestMatchers(HttpMethod.GET, "/**").permitAll();
requests.requestMatchers("/refresh_cache").permitAll();
requests.anyRequest().hasRole("ADMIN");
});
Github github = properties.getGithub();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import java.time.LocalDate;
import java.util.List;

import io.spring.projectapi.github.GithubOperations;
import io.spring.projectapi.ProjectRepository;
import io.spring.projectapi.github.ProjectSupport;
import io.spring.projectapi.web.error.ResourceNotFoundException;
import io.spring.projectapi.web.project.ProjectsController;
Expand Down Expand Up @@ -48,16 +48,16 @@
@ExposesResourceFor(Generation.class)
public class GenerationsController {

private final GithubOperations githubOperations;
private final ProjectRepository projectRepository;

public GenerationsController(GithubOperations githubOperations) {
this.githubOperations = githubOperations;
public GenerationsController(ProjectRepository projectRepository) {
this.projectRepository = projectRepository;
}

@GetMapping
public CollectionModel<EntityModel<Generation>> generations(@PathVariable String id) {
List<ProjectSupport> supports = this.githubOperations.getProjectSupports(id);
String supportPolicy = this.githubOperations.getProjectSupportPolicy(id);
List<ProjectSupport> supports = this.projectRepository.getProjectSupports(id);
String supportPolicy = this.projectRepository.getProjectSupportPolicy(id);
List<Generation> generations = supports.stream()
.map((support) -> asGeneration(support, supportPolicy))
.toList();
Expand All @@ -69,8 +69,8 @@ public CollectionModel<EntityModel<Generation>> generations(@PathVariable String

@GetMapping("/{name}")
public EntityModel<Generation> generation(@PathVariable String id, @PathVariable String name) {
List<ProjectSupport> supports = this.githubOperations.getProjectSupports(id);
String supportPolicy = this.githubOperations.getProjectSupportPolicy(id);
List<ProjectSupport> supports = this.projectRepository.getProjectSupports(id);
String supportPolicy = this.projectRepository.getProjectSupportPolicy(id);
List<Generation> generations = supports.stream()
.map((support) -> asGeneration(support, supportPolicy))
.toList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import java.util.List;

import io.spring.projectapi.github.GithubOperations;
import io.spring.projectapi.ProjectRepository;
import io.spring.projectapi.web.generation.GenerationsController;
import io.spring.projectapi.web.project.Project.Status;
import io.spring.projectapi.web.release.ReleasesController;
Expand Down Expand Up @@ -48,18 +48,18 @@
@ExposesResourceFor(Project.class)
public class ProjectsController {

private final GithubOperations githubOperations;
private final ProjectRepository projectRepository;

private final EntityLinks entityLinks;

public ProjectsController(GithubOperations githubOperations, EntityLinks entityLinks) {
this.githubOperations = githubOperations;
public ProjectsController(ProjectRepository projectRepository, EntityLinks entityLinks) {
this.projectRepository = projectRepository;
this.entityLinks = entityLinks;
}

@GetMapping
public CollectionModel<EntityModel<Project>> projects() throws Exception {
List<Project> projects = this.githubOperations.getProjects().stream().map(this::asProject).toList();
public CollectionModel<EntityModel<Project>> projects() {
List<Project> projects = this.projectRepository.getProjects().stream().map(this::asProject).toList();
CollectionModel<EntityModel<Project>> collection = CollectionModel.of(projects.stream().map((project) -> {
try {
return asModel(project);
Expand All @@ -74,7 +74,7 @@ public CollectionModel<EntityModel<Project>> projects() throws Exception {

@GetMapping("/{id}")
public EntityModel<Project> project(@PathVariable String id) {
Project project = asProject(this.githubOperations.getProject(id));
Project project = asProject(this.projectRepository.getProject(id));
return asModel(project);
}

Expand Down
Loading

0 comments on commit 6b1c70c

Please sign in to comment.