Ninjaws/Quiz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Added] Nightly filling up of fresh data (and swapping), Startup filling of DB, sessionToken for bulk gathering

Browse Source
This commit is contained in:
Ninjaws
2024-11-03 01:43:47 +01:00
parent 121dff910d
commit ec5a1de584
28 changed files with 365 additions and 700 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main/java/com/ninjaws/quiz/services/ApiService.java
+26 -3
View File
@@ -12,6 +12,7 @@ import com.ninjaws.quiz.models.ApiResponse;
import com.ninjaws.quiz.models.CategoryResponse;
import com.ninjaws.quiz.models.QuizSettings;
import com.ninjaws.quiz.models.Request;
import com.ninjaws.quiz.models.SessionTokenDTO;
import jakarta.validation.Valid;
@@ -28,6 +29,7 @@ public class ApiService {
private static final System.Logger logger = System.getLogger(ApiService.class.getName());
private final String apiUrl = "https://opentdb.com";
private String sessionToken;
public ApiService() {
}
@@ -55,21 +57,42 @@ public class ApiService {
* Store them in the DataService
*/
public void collectBulk() {
// TODO: Make this happen only once, when a new database is generated
getCategories();
String url = apiUrl + "/api.php?amount=50";
String tokenStr = sessionToken == null ? "" : "&token="+sessionToken;
String url = apiUrl + "/api.php?amount=50" + tokenStr;
try {
ResponseEntity<String> response = restTemplate.getForEntity(url, String.class);
String json = response.getBody();
ApiResponse apiResponse = cleanerService.extractApiResponse(json);
responseHandler.handleResponse(apiResponse);
/**
* A bit dirty, but otherwise it causes circular dependencies
* Makes sure the categories are refreshed after finishing to replace the questions, to make sure the ids match
* And resets the sessionToken, it will be remade during the next REFRESH
*/
if(apiResponse.getResponseCode() == 4) {
getCategories();
sessionToken = null;
}
} catch (RestClientException | IOException e) {
logger.log(System.Logger.Level.ERROR, "Critical failure", e);
}
}
public void getSessionToken() {
String url = apiUrl + "/api_token.php?command=request";
try {
ResponseEntity<String> response = restTemplate.getForEntity(url, String.class);
String json = response.getBody();
SessionTokenDTO tokenDto = cleanerService.extractSessionToken(json);
sessionToken = tokenDto.getToken();
} catch (RestClientException | IOException e) {
logger.log(System.Logger.Level.ERROR, "Critical failure", e);
}
}
public void getCategories() {
String url = apiUrl + "/api_category.php";
backend/src/main/java/com/ninjaws/quiz/services/CleanerService.java
+37 -1
View File
@@ -10,12 +10,20 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.ninjaws.quiz.entities.Category;
import com.ninjaws.quiz.entities.QuestionCollectingEntity;
import com.ninjaws.quiz.entities.QuestionEntity;
import com.ninjaws.quiz.models.ApiResponse;
import com.ninjaws.quiz.models.CategoryResponse;
import com.ninjaws.quiz.models.QuestionDTO;
import com.ninjaws.quiz.models.SessionTokenDTO;
import com.ninjaws.quiz.repositories.CategoryRepository;
/**
* Deals with conversions from one form of data to another
* Json to Object
* Entity to DTO
* etc
*/
@Service
public class CleanerService {
@@ -44,6 +52,10 @@ public class CleanerService {
return dtos;
}
public SessionTokenDTO extractSessionToken(String json) throws IOException {
return objectMapper.readValue(json, SessionTokenDTO.class);
}
public QuestionDTO questionEntityToDTO(QuestionEntity entity) {
QuestionDTO dto = new QuestionDTO();
dto.setQuestion(entity.getQuestion());
@@ -63,7 +75,7 @@ public class CleanerService {
}
return entities;
}
public QuestionEntity questionDTOtoEntity(QuestionDTO dto) {
QuestionEntity entity = new QuestionEntity();
Category category = categoryRepository.findByName(decodeString(dto.getCategory())).orElseThrow(() -> new IllegalStateException("Category not in DB!"));
@@ -76,6 +88,30 @@ public class CleanerService {
return entity;
}
// TODO: Find a simpler way to convert from one table to another if the data is identical so this nonsense isn't necessary
public List<QuestionCollectingEntity> questionListDTOtoCollectingEntity(List<QuestionDTO> dtos) {
List<QuestionCollectingEntity> entities = new ArrayList<>();
for (int i = 0; i < dtos.size(); i++) {
QuestionDTO dto = dtos.get(i);
entities.add(questionDTOtoCollectingEntity(dto));
}
return entities;
}
public QuestionCollectingEntity questionDTOtoCollectingEntity(QuestionDTO dto) {
QuestionCollectingEntity entity = new QuestionCollectingEntity();
Category category = categoryRepository.findByName(decodeString(dto.getCategory())).orElseThrow(() -> new IllegalStateException("Category not in DB!"));
entity.setCategory(category);
entity.setDifficulty(dto.getDifficulty());
entity.setType(dto.getType());
entity.setQuestion(dto.getQuestion());
entity.setCorrectAnswer(dto.getCorrectAnswer());
entity.setIncorrectAnswers(dto.getIncorrectAnswers());
return entity;
}
/**
* For some reason, the categories come with special characters,
* But the categories from the questions dont
backend/src/main/java/com/ninjaws/quiz/services/DataService.java
+25 -23
View File
@@ -6,17 +6,22 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.ninjaws.quiz.entities.Category;
import com.ninjaws.quiz.entities.QuestionCollectingEntity;
import com.ninjaws.quiz.entities.QuestionEntity;
import com.ninjaws.quiz.models.QuizSettings;
import com.ninjaws.quiz.repositories.CategoryRepository;
import com.ninjaws.quiz.repositories.QuestionCollectingRepository;
import com.ninjaws.quiz.repositories.QuestionRepository;
import jakarta.transaction.Transactional;
@Service
public class DataService {
@Autowired
private QuestionRepository questionRepository;
@Autowired
private QuestionCollectingRepository collectingRepository;
@Autowired
private CategoryRepository categoryRepository;
@@ -32,34 +37,31 @@ public class DataService {
return questionRepository.saveAll(questions);
}
// public List<Question> getRandomQuestions(String type, String category, String difficulty, int limit) {
// return questionRepository.findRandomQuestions(type, limit);
// }
public List<QuestionEntity> lookupQuestions(QuizSettings quizSettings) {
// Long category = Long.valueOf(quizSettings.getCategory().getApiValue() == 1 ? null : quizSettings.getCategory().getApiValue());
// String difficulty = quizSettings.getDifficulty() == QuizSettings.Difficulty.ANY ? null : quizSettings.getDifficulty().getApiValue();
// String type = quizSettings.getType() == QuizSettings.Type.ANY ? null : quizSettings.getType().getApiValue();
// System.out.println(category);
// System.out.println(difficulty);
// System.out.println(type);
return questionRepository.lookupQuestions(quizSettings.getCategory(), quizSettings.getDifficulty(), quizSettings.getType(), quizSettings.getAmount());
}
/**
* Tries to get questions that match the specified QuizSettings
*
* @return The questions if the query was possible, and an empty array if not
*/
// public List<Question> getByQuizSettings(QuizSettings quizSettings) {
// Specification<Product> spec = Specification.where(QuestionSpecification.hasType(type))
// .and(QuestionSpecification.hasCategory(category))
// .and(QuestionSpecification.hasDifficulty(difficulty));
// return productRepository.findAll(spec);
// }
public List<QuestionEntity> lookupQuestions(QuizSettings quizSettings) {
return questionRepository.lookupQuestions(quizSettings.getCategory(), quizSettings.getDifficulty(), quizSettings.getType(), quizSettings.getAmount());
}
public List<Category> saveCategories(List<Category> categories) {
return categoryRepository.saveAll(categories);
}
/**
* Moving the collected questions to the questions list
*
*/
@Transactional
public void migrateQuestions() {
questionRepository.deleteAll();
questionRepository.moveCollection();
collectingRepository.deleteAll();
}
public List<QuestionCollectingEntity> collectQuestions(List<QuestionCollectingEntity> questions) {
return collectingRepository.saveAll(questions);
}
}
backend/src/main/java/com/ninjaws/quiz/services/DatabaseScheduler.java
-20
View File
@@ -1,20 +0,0 @@
package com.ninjaws.quiz.services;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
public class DatabaseScheduler {
@Autowired
private JdbcTemplate secondaryJdbcTemplate;
@Scheduled(fixedRate = 60000) // Runs every 60 seconds
public void createAndFillSecondaryDatabase() {
// Create tables and insert data into the secondary database
secondaryJdbcTemplate.
.execute("CREATE TABLE IF NOT EXISTS sample_table (id INT PRIMARY KEY, name VARCHAR(255))");
secondaryJdbcTemplate.update("INSERT INTO sample_table (id, name) VALUES (?, ?)", 1, "Sample Data");
}
}
backend/src/main/java/com/ninjaws/quiz/services/QuizService.java
+6 -5
View File
@@ -38,7 +38,7 @@ public class QuizService {
/** First, we check if there are already enough items in our db */
List<QuestionEntity> questionsRetrieved = dataService.lookupQuestions(quizSettings);
if(!questionsRetrieved.isEmpty()) {
if(!questionsRetrieved.isEmpty() && questionsRetrieved.size() == quizSettings.getAmount()) {
session.setQuestions(cleanerService.questionListEntitytoDTO(questionsRetrieved));
}
@@ -47,7 +47,7 @@ public class QuizService {
/**
* If the questions were retrieved from the DB, then it doesn't need to be added to the queue
*/
if(!questionsRetrieved.isEmpty()) {
if(!questionsRetrieved.isEmpty() && questionsRetrieved.size() == quizSettings.getAmount()) {
return Optional.of(session.getId());
}
@@ -81,9 +81,10 @@ public class QuizService {
}
/** Anything other than statuscode 0 doesn't result in active sessions, so remove them from the cache after reporting to the frontend */
if(session.getStatusCode() != 0) {
cacheService.removeFromCache(session.getId());
}
// if(session.getStatusCode() != 0) {
/** Regardless of status code, this link is not reusable */
cacheService.removeFromCache(session.getId());
// }
return Optional.of(session);
}
backend/src/main/java/com/ninjaws/quiz/services/ResponseHandlingService.java
+22 -4
View File
@@ -10,6 +10,8 @@ import com.ninjaws.quiz.models.Session;
@Service
public class ResponseHandlingService {
@Autowired
private StateManagementService stateManager;
@Autowired
private CacheService cacheService;
@Autowired
@@ -33,7 +35,8 @@ public class ResponseHandlingService {
case 0:
// Success
updateSessionInCache(session, response);
dataService.saveQuestions(cleanerService.questionListDTOtoEntity(response.getResults()));
/** Managing the token gets difficult since there is a timelimit, so simply dont add these, let the bulk collector do it */
// dataService.saveQuestions(cleanerService.questionListDTOtoEntity(response.getResults()));
break;
case 1:
// Not enough questions to satisfy the request.
@@ -67,11 +70,26 @@ public class ResponseHandlingService {
public void handleResponse(ApiResponse response) {
switch (response.getResponseCode()) {
case 0:
dataService.saveQuestions(cleanerService.questionListDTOtoEntity(response.getResults()));
if(stateManager.getCurrentProcessingState() == StateManagementService.ProcessingState.STARTUP) {
/** Initial saving during the startup */
dataService.saveQuestions(cleanerService.questionListDTOtoEntity(response.getResults()));
} else if (stateManager.getCurrentProcessingState() == StateManagementService.ProcessingState.REFRESH) {
/** Background collecting during the night */
dataService.collectQuestions(cleanerService.questionListDTOtoCollectingEntity(response.getResults()));
}
break;
case 4:
// Signal DataService that we have extracted all the data. Stop gathering, switch to caching-mode
// TODO: Signal DB that it's full, start db-swapping process
// Signal DataService that we have extracted all the data. Stop gathering,
// TODO: Switch to caching-mode
/**
* If it was in the REFRESH state, move the data from collecting to the used questions
* Otherwise simply switch to the NORMAL state
*/
if (stateManager.getCurrentProcessingState() == StateManagementService.ProcessingState.REFRESH) {
dataService.migrateQuestions();
}
stateManager.setCurrentProcessingState(StateManagementService.ProcessingState.NORMAL);
break;
case 5:
// Should be prevented by the scheduler
backend/src/main/java/com/ninjaws/quiz/services/SchedulerService.java
+15 -1
View File
@@ -13,6 +13,8 @@ public class SchedulerService {
private QueueService queueService;
@Autowired
private ApiService apiService;
@Autowired
private StateManagementService stateService;
public SchedulerService() {}
@@ -25,11 +27,23 @@ public class SchedulerService {
@Scheduled(fixedDelay = 5000)
protected void processQueue() {
Request request = queueService.popFromQueue();
if (request != null) {
apiService.handleRequest(request);
} else {
apiService.collectBulk();
if (stateService.getCurrentProcessingState() == StateManagementService.ProcessingState.STARTUP || stateService.getCurrentProcessingState() == StateManagementService.ProcessingState.REFRESH) {
apiService.collectBulk();
}
}
}
/**
* Starts the renewing of the questions, to ensure they stay fresh
*/
@Scheduled(cron = "0 0 1 * * ?")
protected void StartDbRefresh() {
apiService.getSessionToken();
stateService.setCurrentProcessingState(StateManagementService.ProcessingState.REFRESH);
}
}
backend/src/main/java/com/ninjaws/quiz/services/StateManagementService.java
+29
View File
@@ -0,0 +1,29 @@
package com.ninjaws.quiz.services;
import org.springframework.stereotype.Service;
@Service
public class StateManagementService {
/**
* STARTUP: When starting the app
* REFRESH: Scheduled at night, so the database stays fresh
* NORMAL: No special activities
*/
public enum ProcessingState {
STARTUP,
REFRESH,
NORMAL
}
private ProcessingState currentProcessingState;
public ProcessingState getCurrentProcessingState() {
return currentProcessingState;
}
public void setCurrentProcessingState(ProcessingState currentProcessingState) {
this.currentProcessingState = currentProcessingState;
}
}