23 changed files with 67 additions and 219 deletions
-14
View File
@@ -1,14 +0,0 @@
name: Quiz App Deployment
run-name: ${{ gitea.actor }} is deploying Quiz App
on:
push:
branches:
- main
jobs:
Quiz Runner:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
- run: docker compose up -d --build
-13
View File
@@ -1,13 +0,0 @@
name: Quiz App Testing
run-name: ${{ gitea.actor }} is testing Quiz App
on: [push]
jobs:
Quiz Runner:
runs-on: ubuntu-latest
steps:
- run: docker --version
- run: docker compose version
- name: Check out repository code
uses: actions/checkout@v4
# Run tests
-11
View File
@@ -36,14 +36,3 @@ sudo reboot
docker compose up
```
- Visit localhost:4200 to play the quiz!
## Benchmarking
- Go to the /scripts folder and run the LoadTestMultiple.sh script
- Inside you can change how many sessions should be created (set to 6 by default)
- In the /scripts/benchmarks folder you can find some benchmarks I've done
## Notes
- If the data is not present, it will try to query it from the external API, which takes significantly longer (only 1 request per 5 seconds)
- When running it locally, give the backend a few minutes to gather data in bulk (should be completely done in ~7 minutes)
- It refreshes this data every night in the background, so it stays up to date
+5 -38
View File
@@ -1,20 +1,20 @@
# Reflection
# Issues
## Issues during testing (October 30th)
## Issues during testing
### Scheduler
I don't know if it was the scheduler specifically, but whenever I try to test it, both the cache and the queue seem to become empty after the first function is completed.
I don't know how to solve this yet
## Caching solution (October 30th)
## Caching solution
### Lack of data
Not being able to gather blocks of 50 is a big issue. The idea of reusing blocks becomes less viable if I can't just get a big chunk every time for others to use.
Not being able to gather blocks of 50 is a huge issue. The idea of reusing blocks becomes less viable if I can just get a big chunk every time for others to use
I can't force a big chunk, because it might just return a status 1, and then the user gets nothing.
Perhaps I can separate the questions myself and make a collection, that i can build the requests from. However, for every quest tracking if someone has already seen is a huge pain.
## Possible solution (October 31st)
## Possible solution
Cache all data, 50 at a time
Store it in a database
When its done, let the db make blocks for quick cache access
@@ -28,36 +28,3 @@ Then start replacing the cache items one at a time.
Users shouldn't feel this at all, because the active sessions are stored in their own cache items
# Progress so far (November 3rd)
There only seem to be ~4100 items, making the collecting phase only last around 7 minutes.
Swapping dbs is replaced with swapping tables, seems to work just as fine, yet much simpler to set up
TODO: Add a caching state, and start filling up each unique combination with the items from the db
This should lead to performance gains
## Possible future alternative
In the current state (when the cache is done), the backend might hold data 3 times (Cache, active DB table, and background DB table during the night)
### Purely Cached solution
- Every night
- Check for each category+difficulty combination how many questions there are (API has an endpoint for this)
- Track which one you are on, and how many you have
- Take blocks of 50, until the remaining items are less than 50, then get that exact amount
- When they are gathered, either
1. Divide them between 3 cache items: cat:dif:any/multiple/boolean
- (Manually filter on the type, since the API sends questions with either option together), or
2. In one like cat:dif:any and then filter it by hand when a user asks for it
- (This will replace the collection put there the night before)
- Then move to the next one
- Stop when the last combination has been collected
#### When a user requests data:
- Get the items from the cache
- Less items than requested: Inform the user to try a different combination
- More items than requested: Randomize and select a subset
### Tradeoffs
- Advantage: Less total data held on the server
- Downside: Slower gathering, since there are 25+3+4=300 unique combinations (each of which will end with a request of less than 50 items), it will require a decent number requests more than the bulk collecting that the DB does.
@@ -12,7 +12,7 @@ public class WebConfig implements WebMvcConfigurer {
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**") // Allow all endpoints
.allowedOrigins("http://localhost:4200", "https://quiz.ianvink.nl") // Allow specific origin
.allowedOrigins("http://localhost:4200") // Allow specific origin
.allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS") // Allow specific methods
.allowedHeaders("*") // Allow all headers
.allowCredentials(true); // Allow credentials
@@ -1,6 +1,5 @@
package com.ninjaws.quiz.controllers;
import java.util.List;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
@@ -12,11 +11,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.ninjaws.quiz.entities.Category;
import com.ninjaws.quiz.models.Answers;
import com.ninjaws.quiz.models.QuizSettings;
import com.ninjaws.quiz.models.Session;
import com.ninjaws.quiz.services.DataService;
import com.ninjaws.quiz.services.QuizService;
@RestController
@@ -25,8 +22,6 @@ public class QuizController {
@Autowired
private QuizService quizService;
@Autowired
private DataService dataService;
/**
@@ -74,9 +69,4 @@ public class QuizController {
String jsonString = String.format("{\"score\":\"%s\"}", quizService.getScore(answers));
return jsonString;
}
@GetMapping("/categories")
public List<Category> getCategories(){
return dataService.getCategories();
}
}
@@ -50,9 +50,6 @@ public class DataService {
return categoryRepository.saveAll(categories);
}
public List<Category> getCategories(){
return categoryRepository.findAll();
}
/**
* Moving the collected questions to the questions list
*
@@ -24,8 +24,6 @@ public class QuizService {
private DataService dataService;
@Autowired
private CleanerService cleanerService;
@Autowired
private StateManagementService stateManager;
/**
* Actions:
@@ -40,20 +38,19 @@ public class QuizService {
/** First, we check if there are already enough items in our db */
List<QuestionEntity> questionsRetrieved = dataService.lookupQuestions(quizSettings);
/** If the items are not matching the required amount, and the db is not currently being filled up, then they simply don't exist */
if (questionsRetrieved.size() < quizSettings.getAmount() && stateManager.getCurrentProcessingState() != StateManagementService.ProcessingState.STARTUP) {
session.setStatusCode(1);
cacheService.addToCache(session.getId(), session);
return Optional.of(session.getId());
}
/** All items are collected in from the database, it doesn't need to be added to the queue */
if(!questionsRetrieved.isEmpty() && questionsRetrieved.size() == quizSettings.getAmount()) {
session.setQuestions(cleanerService.questionListEntitytoDTO(questionsRetrieved));
cacheService.addToCache(session.getId(), session);
return Optional.of(session.getId());
}
cacheService.addToCache(session.getId(), session);
/**
* If the questions were retrieved from the DB, then it doesn't need to be added to the queue
*/
if(!questionsRetrieved.isEmpty() && questionsRetrieved.size() == quizSettings.getAmount()) {
return Optional.of(session.getId());
}
boolean addedToQueue = queueService.addToQueue(new Request(session.getId(), quizSettings));
/** In this order, because doing the queue before the cache could lead to race conditions (request completion immediately calls cache, but then the item doesn't exist) */
if(!addedToQueue) {
@@ -78,17 +75,16 @@ public class QuizService {
throw new IllegalArgumentException("Invalid session ID: " + sessionId);
}
/** Anything other than statuscode 0 doesn't result in active sessions, so remove them from the cache after reporting to the frontend */
if(session.getStatusCode() != 0) {
cacheService.removeFromCache(session.getId());
return Optional.of(session);
}
/** Still waiting for a response */
if (session.getQuestions() == null) {
return Optional.empty();
}
/** Anything other than statuscode 0 doesn't result in active sessions, so remove them from the cache after reporting to the frontend */
// if(session.getStatusCode() != 0) {
/** Regardless of status code, this link is not reusable */
cacheService.removeFromCache(session.getId());
// }
return Optional.of(session);
}
@@ -97,7 +93,7 @@ public class QuizService {
* Receives: sessionId and answers
*
* Actions:
* - Retrieves session from cache
* - retrieves session from cache
* - Deletes session from cache
* - Matches answers answers to correctAnswers
* - Calculates score
@@ -1,5 +1,4 @@
spring.application.name=quiz
server.address=0.0.0.0
server.port=9090
# H2 Database configuration
@@ -3,7 +3,6 @@ import io.gatling.core.Predef._
import io.gatling.http.Predef._
import scala.concurrent.duration._
/** Does not work, use the LoadTestMultiple.sh in the /scripts folder in the root instead */
class QuestionStatusSimulation extends Simulation {
val httpProtocol = http.baseUrl("http://localhost:9090").acceptHeader("application/json")
-13
View File
@@ -8,11 +8,6 @@ services:
container_name: quiz-frontend-prod-cont
ports:
- "4200:80"
networks:
- quiz-network
restart: always
cpu_period: 100000 # Default is 100,000 microseconds (100ms)
cpu_quota: 5000 # 5% of the total CPU time
backend:
build:
context: ./backend
@@ -22,11 +17,3 @@ services:
container_name: quiz-backend-prod-cont
ports:
- "9090:9090"
networks:
- quiz-network
restart: always
cpu_period: 100000 # Default is 100,000 microseconds (100ms)
cpu_quota: 5000 # 5% of the total CPU time
networks:
quiz-network: # Define the network
-1
View File
@@ -27,6 +27,5 @@ RUN ng build --configuration production
#--- Production
FROM nginx:alpine AS production
COPY --from=build /app/dist/quiz/browser /usr/share/nginx/html
COPY nginx.conf /etc/nginx/conf.d/default.conf
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
-61
View File
@@ -1,61 +0,0 @@
server {
listen 80;
# server_name quiz.ianvink.nl;
location / {
root /usr/share/nginx/html;
index index.html;
try_files $uri $uri/ /index.html;
}
location /api/ {
proxy_set_header Connection '';
proxy_http_version 1.1;
proxy_read_timeout 360s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# add_header 'Access-Control-Allow-Origin' '*'; # Adjust this to your frontend domain
# add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; # Include POST
# add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization'; # Include any other headers
proxy_pass http://backend:9090/;
}
}
# server {
# listen 80;
# server_name quiz.ianvink.nl;
# location /.well-known/acme-challenge/ {
# root /usr/share/nginx/html; # Adjust this if needed
# }
# location / {
# return 301 https://$host$request_uri;
# }
# }
# server {
# listen 443 ssl;
# server_name quiz.example.com;
# ssl_certificate /etc/letsencrypt/live/quiz.example.com/fullchain.pem;
# ssl_certificate_key /etc/letsencrypt/live/quiz.example.com/privkey.pem;
# location / {
# root /usr/share/nginx/html;
# index index.html;
# try_files $uri $uri/ /index.html;
# }
# location /api/ {
# proxy_pass http://backend:9090/; # Adjust this if needed
# proxy_set_header Host $host;
# proxy_set_header X-Real-IP $remote_addr;
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# proxy_set_header X-Forwarded-Proto $scheme;
# }
# }
-4
View File
@@ -1,4 +0,0 @@
export interface Category {
id: number;
name: string;
}
+1 -3
View File
@@ -1,7 +1,5 @@
import { Category } from "./category";
export interface QuizParams {
category: number;
category: string;
type: string;
amount: number;
difficulty: string;
+31 -8
View File
@@ -19,7 +19,6 @@ import { StorageService } from './storage.service';
import { CleanerService } from './cleaner.service';
import { ResponseHandlerService } from './response-handler.service';
import { environment } from '../../../environments/environment';
import { Category } from '../models/category';
@Injectable({
providedIn: 'root',
@@ -41,7 +40,8 @@ export class QuizService {
}
this.storageService.setSession(sessionIdObj.sessionId);
const sessionObj = await lastValueFrom(this.pollQuestions(sessionIdObj.sessionId));
return this.reponseHandler.handleResponse(sessionObj.statusCode, sessionObj);
return this.reponseHandler.handleResponse(sessionIdObj.statusCode, sessionObj);
}
public pollQuestions(sessionId: string) {
@@ -67,7 +67,7 @@ export class QuizService {
private convertParamsToUrl(quizParams: QuizParams) {
let url = `?amount=${quizParams.amount}`;
const catIndex = quizParams.category;
const catIndex = this.getCategories().indexOf(quizParams.category);
url += catIndex !== 0 ? `&category=${catIndex}` : '';
const diff = quizParams.difficulty.toLowerCase();
@@ -83,11 +83,34 @@ export class QuizService {
return url;
}
public async getCategories(): Promise<Category[]> {
let categories: Category[] = await firstValueFrom(this.http.get<any>(this.apiUrl + '/categories'));
categories.push({id:0,name:"Any Category"});
categories.sort((a,b) => a.id - b.id);
return categories;
public getCategories(): string[] {
return [
'Any Category',
'General Knowledge',
'Entertainment: Books',
'Entertainment: Film',
'Entertainment: Music',
'Entertainment: Musicals & Theatres',
'Entertainment: Television',
'Entertainment: Video Games',
'Entertainment: Board Games',
'Science & Nature',
'Science: Computers',
'Science: Mathematics',
'Mythology',
'Sports',
'Geography',
'History',
'Politics',
'Art',
'Celebrities',
'Animals',
'Vehicles',
'Entertainment: Comics',
'Science: Gadgets',
'Entertainment: Japanese Anime & Manga',
'Entertainment: Cartoon & Animations',
];
}
public getDifficulties(): string[] {
@@ -5,7 +5,7 @@
<div class="mb-4">
<label for="category" class="block mb-1">Category</label>
<select id="category" class="w-full p-2 border rounded" [(ngModel)]="category">
<option *ngFor="let category of categories" [value]="category.id">{{ category.name }}</option>
<option *ngFor="let category of quizService.getCategories()" [value]="category">{{ category }}</option>
</select>
</div>
+6 -10
View File
@@ -7,7 +7,6 @@ import { QuizParams } from '../models/quiz-params';
import { SpinnerService } from '../../Spinner/services/spinner.service';
import { CommonModule } from '@angular/common';
import { Question } from '../models/question';
import { Category } from '../models/category';
@Component({
selector: 'app-setup',
@@ -18,8 +17,8 @@ import { Category } from '../models/category';
})
export class SetupComponent {
categories!: Category[];
category!: number;
category: string = "";
difficulty: string = "";
type: string = "";
amount: number = 10;
@@ -29,13 +28,10 @@ export class SetupComponent {
public quizService: QuizService,
private spinnerService: SpinnerService
) {
quizService.getCategories().then((categories: Category[]) => {
/** Setting default values */
this.categories = categories;
this.category = this.categories.find(item => item.id == 0)!.id;
this.difficulty = this.quizService.getDifficulties()[0];
this.type = this.quizService.getTypes()[0];
});
/** Setting default values */
this.category = this.quizService.getCategories()[0];
this.difficulty = this.quizService.getDifficulties()[0];
this.type = this.quizService.getTypes()[0];
}
startQuiz() {
@@ -2,6 +2,5 @@ export const environment = {
production: true,
/** Connecting through the docker compose network is not working for now, so I'll have to use the localhost */
// apiUrl: 'http://backend:9090',
apiUrl: '/api' //'https://api.nosotros.ianvink.nl',
// apiUrl: 'http://localhost:9090',
apiUrl: 'http://localhost:9090',
};
+4 -3
View File
@@ -2,15 +2,16 @@
start_time=$(date +%s)
for i in {1..6}; do
for i in {1..1000}; do
./LoadTestOne-Silent.sh &
# sleep 0.1
done
wait
wait # Wait for all background jobs to finish
end_time=$(date +%s)
end_time=$(date +%s) # Capture the end time
# Calculate and print the elapsed time
elapsed_time=$((end_time - start_time))
echo "Done!"
echo "Total time taken: $elapsed_time seconds"

Before

Width:  |  Height:  |  Size: 62 KiB

After

Width:  |  Height:  |  Size: 62 KiB

Before

Width:  |  Height:  |  Size: 68 KiB

After

Width:  |  Height:  |  Size: 68 KiB