refactor and remove internal file
This commit is contained in:
Elshimy Ziad Magdy Taha
parent
0ab6954231
commit
0e1b64b7b9
29 changed files with 43 additions and 1765 deletions
|
|
@ -5,10 +5,10 @@ import (
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/jobs-scraper/api/commands/job"
|
"github.com/jobs-scraper/api/commands/job"
|
||||||
"github.com/jobs-scraper/infrastructure/rabbitmq"
|
"github.com/jobs-scraper/shared/infrastructure/rabbitmq"
|
||||||
"github.com/jobs-scraper/internal/ports"
|
"github.com/jobs-scraper/shared/ports"
|
||||||
"github.com/jobs-scraper/internal/repo"
|
"github.com/jobs-scraper/shared/repo"
|
||||||
"github.com/jobs-scraper/internal/server"
|
"github.com/jobs-scraper/shared/server"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Application struct {
|
type Application struct {
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,10 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
"github.com/jobs-scraper/infrastructure/rabbitmq"
|
"github.com/jobs-scraper/shared/domain"
|
||||||
"github.com/jobs-scraper/internal/domain"
|
"github.com/jobs-scraper/shared/infrastructure/rabbitmq"
|
||||||
"github.com/jobs-scraper/internal/ports"
|
"github.com/jobs-scraper/shared/ports"
|
||||||
"github.com/jobs-scraper/internal/repo"
|
"github.com/jobs-scraper/shared/repo"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CreateJob struct {
|
type CreateJob struct {
|
||||||
|
|
|
||||||
12
api/main.go
12
api/main.go
|
|
@ -11,11 +11,11 @@ import (
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/jobs-scraper/api/app"
|
"github.com/jobs-scraper/api/app"
|
||||||
_ "github.com/jobs-scraper/docs"
|
"github.com/jobs-scraper/shared/infrastructure"
|
||||||
"github.com/jobs-scraper/infrastructure"
|
httpHandler "github.com/jobs-scraper/shared/infrastructure/http"
|
||||||
httpHandler "github.com/jobs-scraper/infrastructure/http"
|
"github.com/jobs-scraper/shared/infrastructure/rabbitmq"
|
||||||
"github.com/jobs-scraper/infrastructure/rabbitmq"
|
"github.com/jobs-scraper/shared/repo"
|
||||||
"github.com/jobs-scraper/internal/repo"
|
_ "github.com/jobs-scraper/swagger"
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
httpSwagger "github.com/swaggo/http-swagger"
|
httpSwagger "github.com/swaggo/http-swagger"
|
||||||
)
|
)
|
||||||
|
|
@ -72,7 +72,7 @@ func main() {
|
||||||
|
|
||||||
// Create job analysis result repository
|
// Create job analysis result repository
|
||||||
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
|
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
|
||||||
|
|
||||||
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo)
|
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo)
|
||||||
|
|
||||||
jobHandler.RegisterRoutes(router)
|
jobHandler.RegisterRoutes(router)
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
"github.com/jobs-scraper/cron"
|
"github.com/jobs-scraper/cron"
|
||||||
"github.com/jobs-scraper/infrastructure"
|
"github.com/jobs-scraper/shared/infrastructure"
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -32,7 +32,7 @@ func main() {
|
||||||
log.Println("Successfully connected to db")
|
log.Println("Successfully connected to db")
|
||||||
|
|
||||||
analyzer := cron.NewJobAnalyzer(db)
|
analyzer := cron.NewJobAnalyzer(db)
|
||||||
|
|
||||||
if err := analyzer.AnalyzeJobs(); err != nil {
|
if err := analyzer.AnalyzeJobs(); err != nil {
|
||||||
log.Fatalf("Error analyzing jobs: %v", err)
|
log.Fatalf("Error analyzing jobs: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ package main
|
||||||
import (
|
import (
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
"github.com/jobs-scraper/infrastructure"
|
"github.com/jobs-scraper/shared/infrastructure"
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,19 +6,19 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/jobs-scraper/infrastructure/rabbitmq"
|
"github.com/jobs-scraper/shared/domain"
|
||||||
"github.com/jobs-scraper/internal/domain"
|
"github.com/jobs-scraper/shared/infrastructure/rabbitmq"
|
||||||
"github.com/jobs-scraper/internal/repo"
|
"github.com/jobs-scraper/shared/openrouter"
|
||||||
"github.com/jobs-scraper/internal/services"
|
"github.com/jobs-scraper/shared/repo"
|
||||||
)
|
)
|
||||||
|
|
||||||
type JobAnalyzer struct {
|
type JobAnalyzer struct {
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
jobRepo *repo.JobRepository
|
jobRepo *repo.JobRepository
|
||||||
jobDescriptionRepo *repo.JobDescriptionRepository
|
jobDescriptionRepo *repo.JobDescriptionRepository
|
||||||
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
|
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
|
||||||
openRouterService services.OpenRouterService
|
openRouterService services.OpenRouterService
|
||||||
rmq *rabbitmq.RabbitMQClient
|
rmq *rabbitmq.RabbitMQClient
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewJobAnalyzer(db *sql.DB) *JobAnalyzer {
|
func NewJobAnalyzer(db *sql.DB) *JobAnalyzer {
|
||||||
|
|
@ -122,7 +122,7 @@ func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error {
|
||||||
message := rabbitmq.CvAnalyzeMessage{
|
message := rabbitmq.CvAnalyzeMessage{
|
||||||
JobID: job.ID,
|
JobID: job.ID,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ja.publishCvAnalysisMessage(message); err != nil {
|
if err := ja.publishCvAnalysisMessage(message); err != nil {
|
||||||
log.Printf("Warning: Failed to publish CV analysis message: %v", err)
|
log.Printf("Warning: Failed to publish CV analysis message: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,10 +4,10 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/jobs-scraper/infrastructure"
|
"github.com/jobs-scraper/shared/domain"
|
||||||
"github.com/jobs-scraper/internal/domain"
|
"github.com/jobs-scraper/shared/infrastructure"
|
||||||
"github.com/jobs-scraper/internal/repo"
|
"github.com/jobs-scraper/shared/openrouter"
|
||||||
"github.com/jobs-scraper/internal/services"
|
"github.com/jobs-scraper/shared/repo"
|
||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
1
go.work
1
go.work
|
|
@ -7,5 +7,6 @@ use (
|
||||||
./shared/infrastructure
|
./shared/infrastructure
|
||||||
./shared/ports
|
./shared/ports
|
||||||
./shared/repo
|
./shared/repo
|
||||||
|
./shared/server
|
||||||
./shared/utils
|
./shared/utils
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,171 +0,0 @@
|
||||||
package infrastructure
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
|
|
||||||
"github.com/golang-migrate/migrate/v4"
|
|
||||||
"github.com/golang-migrate/migrate/v4/database/postgres"
|
|
||||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
|
||||||
_ "github.com/lib/pq"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Config holds database configuration
|
|
||||||
type Config struct {
|
|
||||||
Host string
|
|
||||||
Port string
|
|
||||||
User string
|
|
||||||
Password string
|
|
||||||
DBName string
|
|
||||||
SSLMode string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewConnection creates a new database connection
|
|
||||||
func NewConnection(config Config) (*sql.DB, error) {
|
|
||||||
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
|
|
||||||
config.Host, config.Port, config.User, config.Password, config.DBName, config.SSLMode)
|
|
||||||
|
|
||||||
db, err := sql.Open("postgres", dsn)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to open database connection: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := db.Ping(); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to ping database: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return db, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadConfigFromEnv loads database configuration from environment variables
|
|
||||||
func LoadConfigFromEnv() Config {
|
|
||||||
return Config{
|
|
||||||
Host: getEnv("DB_HOST", "localhost"),
|
|
||||||
Port: getEnv("DB_PORT", "5432"),
|
|
||||||
User: getEnv("DB_USER", "postgres"),
|
|
||||||
Password: getEnv("DB_PASSWORD", "password"),
|
|
||||||
DBName: getEnv("DB_NAME", "linkedin_jobs"),
|
|
||||||
SSLMode: getEnv("DB_SSLMODE", "disable"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEnv(key, defaultValue string) string {
|
|
||||||
if value := os.Getenv(key); value != "" {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
return defaultValue
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunMigrations runs all pending migrations
|
|
||||||
func RunMigrations(db *sql.DB) error {
|
|
||||||
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create postgres driver: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the migrations directory path
|
|
||||||
migrationsPath, err := getMigrationsPath()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to get migrations path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
m, err := migrate.NewWithDatabaseInstance(
|
|
||||||
fmt.Sprintf("file://%s", migrationsPath),
|
|
||||||
"postgres",
|
|
||||||
driver,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create migrate instance: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
|
||||||
return fmt.Errorf("failed to run migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RollbackMigrations rolls back migrations by the specified number of steps
|
|
||||||
func RollbackMigrations(db *sql.DB, steps int) error {
|
|
||||||
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create postgres driver: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
migrationsPath, err := getMigrationsPath()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to get migrations path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
m, err := migrate.NewWithDatabaseInstance(
|
|
||||||
fmt.Sprintf("file://%s", migrationsPath),
|
|
||||||
"postgres",
|
|
||||||
driver,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create migrate instance: %w", err)
|
|
||||||
}
|
|
||||||
defer m.Close()
|
|
||||||
|
|
||||||
if err := m.Steps(-steps); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
|
||||||
return fmt.Errorf("failed to rollback migrations: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetMigrationVersion returns the current migration version
|
|
||||||
func GetMigrationVersion(db *sql.DB) (uint, bool, error) {
|
|
||||||
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
|
||||||
if err != nil {
|
|
||||||
return 0, false, fmt.Errorf("failed to create postgres driver: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
migrationsPath, err := getMigrationsPath()
|
|
||||||
if err != nil {
|
|
||||||
return 0, false, fmt.Errorf("failed to get migrations path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
m, err := migrate.NewWithDatabaseInstance(
|
|
||||||
fmt.Sprintf("file://%s", migrationsPath),
|
|
||||||
"postgres",
|
|
||||||
driver,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, false, fmt.Errorf("failed to create migrate instance: %w", err)
|
|
||||||
}
|
|
||||||
defer m.Close()
|
|
||||||
|
|
||||||
version, dirty, err := m.Version()
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, migrate.ErrNilVersion) {
|
|
||||||
return 0, false, nil
|
|
||||||
}
|
|
||||||
return 0, false, fmt.Errorf("failed to get migration version: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return version, dirty, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func getMigrationsPath() (string, error) {
|
|
||||||
// Get the directory of the current source file
|
|
||||||
_, filename, _, ok := runtime.Caller(0)
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("failed to get current file path")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the project root (go up from infrastructure/ to project root)
|
|
||||||
projectRoot := filepath.Dir(filepath.Dir(filename))
|
|
||||||
migrationsPath := filepath.Join(projectRoot, "migrations")
|
|
||||||
|
|
||||||
// Verify the migrations directory exists
|
|
||||||
if _, err := os.Stat(migrationsPath); err != nil {
|
|
||||||
return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return migrationsPath, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,148 +0,0 @@
|
||||||
package http
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/jobs-scraper/internal/ports"
|
|
||||||
"github.com/jobs-scraper/internal/repo"
|
|
||||||
"github.com/jobs-scraper/internal/utils"
|
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
|
||||||
)
|
|
||||||
|
|
||||||
// JobHandler handles HTTP requests related to jobs
|
|
||||||
type JobHandler struct {
|
|
||||||
jobCommands *ports.JobCommands
|
|
||||||
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewJobHandler creates a new instance of JobHandler
|
|
||||||
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler {
|
|
||||||
return &JobHandler{
|
|
||||||
jobCommands: jobCommands,
|
|
||||||
jobAnalysisResultRepo: jobAnalysisResultRepo,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RegisterRoutes registers all routes to the router
|
|
||||||
func (h *JobHandler) RegisterRoutes(router *mux.Router) {
|
|
||||||
jobs := router.PathPrefix("/jobs").Subrouter()
|
|
||||||
|
|
||||||
//jobs.Use(AuthMiddleware)
|
|
||||||
|
|
||||||
jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST")
|
|
||||||
jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET")
|
|
||||||
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
|
|
||||||
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateJob handles the creation of a new job
|
|
||||||
// @Summary Create a new job
|
|
||||||
// @Description Creates a new job with the provided specifications
|
|
||||||
// @Tags jobs
|
|
||||||
// @Accept json
|
|
||||||
// @Produce json
|
|
||||||
// @Param job body ports.CreateJobCommand true "Job creation data"
|
|
||||||
// @Success 201 {object} map[string]string "Successfully created job"
|
|
||||||
// @Failure 400 {object} map[string]string "Invalid request data"
|
|
||||||
// @Failure 500 {object} map[string]string "Internal server error"
|
|
||||||
// @Router /jobs [post]
|
|
||||||
func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
|
|
||||||
var cmd ports.CreateJobCommand
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil {
|
|
||||||
slog.Error(err.Error())
|
|
||||||
return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON())
|
|
||||||
}
|
|
||||||
|
|
||||||
err := h.jobCommands.CreateJob.Handle(cmd)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error(err.Error())
|
|
||||||
return utils.NewAPIError(http.StatusInternalServerError, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetAllAnalysisResults handles retrieving all job analysis results
|
|
||||||
// @Summary Get all job analysis results
|
|
||||||
// @Description Retrieves all job analysis results from the database
|
|
||||||
// @Tags jobs
|
|
||||||
// @Produce json
|
|
||||||
// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results"
|
|
||||||
// @Failure 500 {object} map[string]string "Internal server error"
|
|
||||||
// @Router /jobs/analysis [get]
|
|
||||||
func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error {
|
|
||||||
results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults()
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("Failed to get analysis results", "error", err)
|
|
||||||
return utils.NewAPIError(http.StatusInternalServerError, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
utils.WriteJSON(w, http.StatusOK, results)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetJobAnalysisResult handles retrieving analysis result for a specific job
|
|
||||||
// @Summary Get job analysis result by job ID
|
|
||||||
// @Description Retrieves the analysis result for a specific job
|
|
||||||
// @Tags jobs
|
|
||||||
// @Produce json
|
|
||||||
// @Param id path int true "Job ID"
|
|
||||||
// @Success 200 {object} domain.JobAnalysisResult "Job analysis result"
|
|
||||||
// @Failure 400 {object} map[string]string "Invalid job ID"
|
|
||||||
// @Failure 404 {object} map[string]string "Analysis result not found"
|
|
||||||
// @Failure 500 {object} map[string]string "Internal server error"
|
|
||||||
// @Router /jobs/{id}/analysis [get]
|
|
||||||
func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error {
|
|
||||||
vars := mux.Vars(r)
|
|
||||||
jobID, err := strconv.ParseInt(vars["id"], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("Failed to get analysis result", "jobID", jobID, "error", err)
|
|
||||||
return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found"))
|
|
||||||
}
|
|
||||||
|
|
||||||
utils.WriteJSON(w, http.StatusOK, result)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetTopMatches handles retrieving top matching jobs based on analysis score
|
|
||||||
// @Summary Get top matching jobs
|
|
||||||
// @Description Retrieves jobs with high analysis match scores
|
|
||||||
// @Tags jobs
|
|
||||||
// @Produce json
|
|
||||||
// @Param min_score query int false "Minimum match score (default: 70)"
|
|
||||||
// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results"
|
|
||||||
// @Failure 400 {object} map[string]string "Invalid minimum score"
|
|
||||||
// @Failure 500 {object} map[string]string "Internal server error"
|
|
||||||
// @Router /jobs/analysis/top-matches [get]
|
|
||||||
func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error {
|
|
||||||
minScoreStr := r.URL.Query().Get("min_score")
|
|
||||||
minScore := 70 // default minimum score
|
|
||||||
|
|
||||||
if minScoreStr != "" {
|
|
||||||
var err error
|
|
||||||
minScore, err = strconv.Atoi(minScoreStr)
|
|
||||||
if err != nil {
|
|
||||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore)
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("Failed to get top matches", "minScore", minScore, "error", err)
|
|
||||||
return utils.NewAPIError(http.StatusInternalServerError, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
utils.WriteJSON(w, http.StatusOK, results)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,143 +0,0 @@
|
||||||
package http
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CORSMiddleware handles Cross-Origin Resource Sharing
|
|
||||||
func CORSMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// Set CORS headers
|
|
||||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
|
||||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
|
||||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
|
|
||||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
|
||||||
w.Header().Set("Access-Control-Max-Age", "86400")
|
|
||||||
|
|
||||||
// Handle preflight OPTIONS requests
|
|
||||||
if r.Method == "OPTIONS" {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogsMiddleware logs HTTP requests with timing information
|
|
||||||
func LogsMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
start := time.Now()
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// AuthMiddleware handles JWT token validation, API keys, and basic auth
|
|
||||||
func AuthMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// Skip auth for swagger endpoints and health checks
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" {
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try Bearer token authentication first
|
|
||||||
authHeader := r.Header.Get("Authorization")
|
|
||||||
if authHeader != "" {
|
|
||||||
if strings.HasPrefix(authHeader, "Bearer ") {
|
|
||||||
token := strings.TrimPrefix(authHeader, "Bearer ")
|
|
||||||
if token != "" {
|
|
||||||
userID, err := validateJWTToken(token)
|
|
||||||
if err == nil {
|
|
||||||
ctx := context.WithValue(r.Context(), "user_id", userID)
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try API Key authentication
|
|
||||||
apiKey := r.Header.Get("X-API-Key")
|
|
||||||
if apiKey != "" {
|
|
||||||
if isValidAPIKey(apiKey) {
|
|
||||||
ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey))
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try Basic authentication
|
|
||||||
username, password, ok := r.BasicAuth()
|
|
||||||
if ok {
|
|
||||||
if isValidCredentials(username, password) {
|
|
||||||
ctx := context.WithValue(r.Context(), "username", username)
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to cookie-based auth (existing behavior)
|
|
||||||
token, err := r.Cookie("token")
|
|
||||||
if err == nil && token.Value == "123" {
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// No valid authentication found
|
|
||||||
response := map[string]interface{}{
|
|
||||||
"error": "Unauthorized",
|
|
||||||
"message": "Valid authentication required. Use Bearer token, API key, or basic auth.",
|
|
||||||
"code": http.StatusUnauthorized,
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
|
||||||
json.NewEncoder(w).Encode(response)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper functions for authentication validation
|
|
||||||
func validateJWTToken(token string) (string, error) {
|
|
||||||
// TODO: Implement JWT validation logic with your preferred JWT library
|
|
||||||
// For demonstration, accept a demo token
|
|
||||||
if token == "valid-jwt-token" {
|
|
||||||
return "user-123", nil
|
|
||||||
}
|
|
||||||
return "", http.ErrAbortHandler
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidAPIKey(apiKey string) bool {
|
|
||||||
// TODO: Implement API key validation logic
|
|
||||||
// For demonstration, accept a demo API key
|
|
||||||
return apiKey == "demo-api-key-123"
|
|
||||||
}
|
|
||||||
|
|
||||||
func getUserIDFromAPIKey(apiKey string) string {
|
|
||||||
// TODO: Implement logic to get user ID from API key
|
|
||||||
// For demonstration, return a mock user ID
|
|
||||||
return "api-user-123"
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidCredentials(username, password string) bool {
|
|
||||||
// TODO: Implement credential validation logic
|
|
||||||
// For demonstration, accept demo credentials
|
|
||||||
return username == "admin" && password == "password"
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetUserIDFromContext extracts user ID from request context
|
|
||||||
func GetUserIDFromContext(ctx context.Context) (string, bool) {
|
|
||||||
userID, ok := ctx.Value("user_id").(string)
|
|
||||||
return userID, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetUsernameFromContext extracts username from request context
|
|
||||||
func GetUsernameFromContext(ctx context.Context) (string, bool) {
|
|
||||||
username, ok := ctx.Value("username").(string)
|
|
||||||
return username, ok
|
|
||||||
}
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
package rabbitmq
|
|
||||||
|
|
||||||
type CvAnalyzeMessage struct {
|
|
||||||
JobID int64 `json:"jobID"`
|
|
||||||
}
|
|
||||||
|
|
@ -1,248 +0,0 @@
|
||||||
package rabbitmq
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
ScraperExchange = "scraper_exchange"
|
|
||||||
LinkedInQueue = "scraper.linkedin"
|
|
||||||
IndeedQueue = "scraper.indeed"
|
|
||||||
BaytQueue = "scraper.bayt"
|
|
||||||
TokyoDevQueue = "scraper.tokyodev"
|
|
||||||
GlassDoorQueue = "scraper.glassdoor"
|
|
||||||
JapanDevQueue = "scraper.japandev"
|
|
||||||
DeadLetterExchange = "scraper_dlx"
|
|
||||||
CvAnalyzeExchange = "cv_exchange"
|
|
||||||
CvAnalyzeQueue = "cv.analyze"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RabbitMQClient struct {
|
|
||||||
Conn *amqp.Connection
|
|
||||||
Channel *amqp.Channel
|
|
||||||
}
|
|
||||||
|
|
||||||
type MessageHandler func([]byte) error
|
|
||||||
|
|
||||||
func NewRabbitMQClient() (*RabbitMQClient, error) {
|
|
||||||
// Get RabbitMQ URL from environment or use default
|
|
||||||
rabbitmqURL := os.Getenv("RABBITMQ_URL")
|
|
||||||
if rabbitmqURL == "" {
|
|
||||||
rabbitmqURL = "amqp://guest:guest@localhost:5672/"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connect to RabbitMQ with retry logic
|
|
||||||
var conn *amqp.Connection
|
|
||||||
var err error
|
|
||||||
|
|
||||||
for i := range 5 {
|
|
||||||
conn, err = amqp.Dial(rabbitmqURL)
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
log.Printf("Failed to connect to RabbitMQ (attempt %d/5): %v", i+1, err)
|
|
||||||
time.Sleep(5 * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error connecting to RabbitMQ after 5 attempts: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a channel
|
|
||||||
ch, err := conn.Channel()
|
|
||||||
if err != nil {
|
|
||||||
conn.Close()
|
|
||||||
return nil, fmt.Errorf("error creating channel: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
client := &RabbitMQClient{
|
|
||||||
Conn: conn,
|
|
||||||
Channel: ch,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Setup exchanges and queues
|
|
||||||
if err := client.setupInfrastructure(); err != nil {
|
|
||||||
client.Close()
|
|
||||||
return nil, fmt.Errorf("error setting up infrastructure: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("Successfully connected to RabbitMQ and set up infrastructure")
|
|
||||||
return client, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *RabbitMQClient) setupInfrastructure() error {
|
|
||||||
// Declare dead letter exchange
|
|
||||||
err := r.Channel.ExchangeDeclare(
|
|
||||||
DeadLetterExchange,
|
|
||||||
"direct",
|
|
||||||
true, // durable
|
|
||||||
false, // auto-delete
|
|
||||||
false, // internal
|
|
||||||
false, // no-wait
|
|
||||||
nil, // arguments
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error declaring dead letter exchange: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Declare main exchange
|
|
||||||
err = r.Channel.ExchangeDeclare(
|
|
||||||
ScraperExchange,
|
|
||||||
"topic",
|
|
||||||
true, // durable
|
|
||||||
false, // auto-delete
|
|
||||||
false, // internal
|
|
||||||
false, // no-wait
|
|
||||||
nil, // arguments
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error declaring exchange: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Define queues with their routing keys
|
|
||||||
queues := map[string]string{
|
|
||||||
LinkedInQueue: "scraper.linkedin",
|
|
||||||
IndeedQueue: "scraper.indeed",
|
|
||||||
BaytQueue: "scraper.bayt",
|
|
||||||
TokyoDevQueue: "scraper.tokyodev",
|
|
||||||
JapanDevQueue: "scraper.japandev",
|
|
||||||
GlassDoorQueue: "scraper.glassdoor",
|
|
||||||
}
|
|
||||||
|
|
||||||
// Declare queues with dead letter exchange and TTL
|
|
||||||
for queueName, routingKey := range queues {
|
|
||||||
dlqName := queueName + "_dlq"
|
|
||||||
_, err := r.Channel.QueueDeclare(
|
|
||||||
dlqName,
|
|
||||||
true, // durable
|
|
||||||
false, // delete when unused
|
|
||||||
false, // exclusive
|
|
||||||
false, // no-wait
|
|
||||||
nil, // arguments
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error declaring dead letter queue %s: %v", dlqName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = r.Channel.QueueBind(
|
|
||||||
dlqName,
|
|
||||||
queueName, // routing key is the original queue name
|
|
||||||
DeadLetterExchange,
|
|
||||||
false,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error binding dead letter queue %s: %v", dlqName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = r.Channel.QueueDeclare(
|
|
||||||
queueName,
|
|
||||||
true, // durable
|
|
||||||
false, // delete when unused
|
|
||||||
false, // exclusive
|
|
||||||
false, // no-wait
|
|
||||||
amqp.Table{
|
|
||||||
"x-dead-letter-exchange": DeadLetterExchange,
|
|
||||||
"x-dead-letter-routing-key": queueName,
|
|
||||||
"x-message-ttl": int64(24 * time.Hour / time.Millisecond), // 24 hours TTL
|
|
||||||
"x-max-retries": 3,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error declaring queue %s: %v", queueName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = r.Channel.QueueBind(
|
|
||||||
queueName,
|
|
||||||
routingKey,
|
|
||||||
ScraperExchange,
|
|
||||||
false,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error binding queue %s: %v", queueName, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Publish publishes a message to a specific routing key
|
|
||||||
func (r *RabbitMQClient) Publish(routingKey, exchange string, data []byte) error {
|
|
||||||
log.Printf("Publishing to exchange=%s, routingKey=%s, data=%s", ScraperExchange, routingKey, string(data))
|
|
||||||
err := r.Channel.Publish(
|
|
||||||
ScraperExchange,
|
|
||||||
routingKey,
|
|
||||||
false,
|
|
||||||
false,
|
|
||||||
amqp.Publishing{
|
|
||||||
ContentType: "application/json",
|
|
||||||
Body: data,
|
|
||||||
DeliveryMode: amqp.Persistent,
|
|
||||||
Timestamp: time.Now().UTC(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error publishing: %v", err)
|
|
||||||
return fmt.Errorf("error publishing message: %v", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subscribe creates a subscription to a specific queue with production-ready error handling
|
|
||||||
func (r *RabbitMQClient) Subscribe(queueName, consumerName string, handler MessageHandler) error {
|
|
||||||
err := r.Channel.Qos(
|
|
||||||
1, // prefetch count
|
|
||||||
0, // prefetch size
|
|
||||||
false, // global
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error setting QoS: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start consuming messages
|
|
||||||
msgs, err := r.Channel.Consume(
|
|
||||||
queueName,
|
|
||||||
consumerName,
|
|
||||||
false, // auto-ack
|
|
||||||
false, // exclusive
|
|
||||||
false, // no-local
|
|
||||||
false, // no-wait
|
|
||||||
nil, // args
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error starting consumer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for msg := range msgs {
|
|
||||||
// Call the handler
|
|
||||||
err := handler(msg.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error processing message: %v", err)
|
|
||||||
// Reject and requeue the message (will go to DLX after max retries)
|
|
||||||
msg.Nack(false, true)
|
|
||||||
} else {
|
|
||||||
msg.Ack(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
log.Printf("Successfully subscribed to queue: %s ", queueName)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the RabbitMQ connection and channel
|
|
||||||
func (r *RabbitMQClient) Close() {
|
|
||||||
if r.Channel != nil {
|
|
||||||
r.Channel.Close()
|
|
||||||
}
|
|
||||||
if r.Conn != nil {
|
|
||||||
r.Conn.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
package domain
|
|
||||||
|
|
||||||
type JobDescription struct {
|
|
||||||
JobID int64
|
|
||||||
Description string
|
|
||||||
Criteria map[string]string
|
|
||||||
}
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
package domain
|
|
||||||
|
|
||||||
type JobWithDescription struct {
|
|
||||||
Job Job
|
|
||||||
JobDescription JobDescription
|
|
||||||
}
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
package domain
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// JobProvider represents different job board providers
|
|
||||||
type JobProvider int
|
|
||||||
|
|
||||||
type JobStatus int
|
|
||||||
|
|
||||||
const (
|
|
||||||
LinkedIn JobProvider = iota
|
|
||||||
Indeed
|
|
||||||
Glassdoor
|
|
||||||
Bayt
|
|
||||||
TokyoDev
|
|
||||||
JapanDev
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
JobStatusCreated JobStatus = iota
|
|
||||||
JobStatusAnalyzed
|
|
||||||
JobStatusError
|
|
||||||
)
|
|
||||||
|
|
||||||
type Job struct {
|
|
||||||
ID int64
|
|
||||||
Title string
|
|
||||||
Company string
|
|
||||||
CompanyLink string
|
|
||||||
Location string
|
|
||||||
JobLink string
|
|
||||||
Provider JobProvider
|
|
||||||
JobPostTime *time.Time
|
|
||||||
Status JobStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
type JobAnalysisResult struct {
|
|
||||||
ID int64
|
|
||||||
JobID int64
|
|
||||||
AnalysisResult string
|
|
||||||
MatchScore *int
|
|
||||||
KeySkills []string
|
|
||||||
MissingSkills []string
|
|
||||||
Recommendations *string
|
|
||||||
AnalyzedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type SearchQuery struct {
|
|
||||||
Keywords string `json:"keywords"`
|
|
||||||
Location string `json:"location"`
|
|
||||||
NumPages int `json:"numPages"`
|
|
||||||
FWT string `json:"f_WT"` // Work type filter (1=onsite, 2=remote, 3=hybrid)
|
|
||||||
}
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
package ports
|
|
||||||
|
|
||||||
import "github.com/jobs-scraper/internal/domain"
|
|
||||||
|
|
||||||
type JobCommands struct {
|
|
||||||
CreateJob JobCommandHandler
|
|
||||||
}
|
|
||||||
|
|
||||||
// CreateJobCommand represents the command to create a job
|
|
||||||
type CreateJobCommand struct {
|
|
||||||
Location string
|
|
||||||
Keywords string
|
|
||||||
FWT string
|
|
||||||
Provider domain.JobProvider
|
|
||||||
NumPages int
|
|
||||||
}
|
|
||||||
|
|
||||||
// JobCommandHandler defines the interface for handling job commands
|
|
||||||
type JobCommandHandler interface {
|
|
||||||
Handle(cmd CreateJobCommand) error
|
|
||||||
}
|
|
||||||
|
|
@ -1,152 +0,0 @@
|
||||||
package repo
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/jobs-scraper/internal/domain"
|
|
||||||
"github.com/lib/pq"
|
|
||||||
)
|
|
||||||
|
|
||||||
type JobAnalysisResultRepository struct {
|
|
||||||
db *sql.DB
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewJobAnalysisResultRepository(db *sql.DB) *JobAnalysisResultRepository {
|
|
||||||
return &JobAnalysisResultRepository{db: db}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobAnalysisResultRepository) SaveAnalysisResult(result *domain.JobAnalysisResult) error {
|
|
||||||
query := `
|
|
||||||
INSERT INTO job_analysis_results (job_id, analysis_result, match_score, key_skills, missing_skills, recommendations)
|
|
||||||
VALUES ($1, $2, $3, $4, $5, $6)
|
|
||||||
RETURNING id, analyzed_at
|
|
||||||
`
|
|
||||||
|
|
||||||
err := r.db.QueryRow(
|
|
||||||
query,
|
|
||||||
result.JobID,
|
|
||||||
result.AnalysisResult,
|
|
||||||
result.MatchScore,
|
|
||||||
pq.Array(result.KeySkills),
|
|
||||||
pq.Array(result.MissingSkills),
|
|
||||||
result.Recommendations,
|
|
||||||
).Scan(&result.ID, &result.AnalyzedAt)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error saving job analysis result: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobAnalysisResultRepository) GetAnalysisResultByJobID(jobID int64) (*domain.JobAnalysisResult, error) {
|
|
||||||
query := `
|
|
||||||
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
|
|
||||||
FROM job_analysis_results
|
|
||||||
WHERE job_id = $1
|
|
||||||
ORDER BY analyzed_at DESC
|
|
||||||
LIMIT 1
|
|
||||||
`
|
|
||||||
|
|
||||||
var result domain.JobAnalysisResult
|
|
||||||
err := r.db.QueryRow(query, jobID).Scan(
|
|
||||||
&result.ID,
|
|
||||||
&result.JobID,
|
|
||||||
&result.AnalysisResult,
|
|
||||||
&result.MatchScore,
|
|
||||||
pq.Array(&result.KeySkills),
|
|
||||||
pq.Array(&result.MissingSkills),
|
|
||||||
&result.Recommendations,
|
|
||||||
&result.AnalyzedAt,
|
|
||||||
)
|
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
return nil, fmt.Errorf("no analysis result found for job ID %d", jobID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error querying job analysis result: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobAnalysisResultRepository) GetAllAnalysisResults() ([]domain.JobAnalysisResult, error) {
|
|
||||||
query := `
|
|
||||||
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
|
|
||||||
FROM job_analysis_results
|
|
||||||
ORDER BY analyzed_at DESC
|
|
||||||
`
|
|
||||||
|
|
||||||
rows, err := r.db.Query(query)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error querying job analysis results: %v", err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var results []domain.JobAnalysisResult
|
|
||||||
for rows.Next() {
|
|
||||||
var result domain.JobAnalysisResult
|
|
||||||
err := rows.Scan(
|
|
||||||
&result.ID,
|
|
||||||
&result.JobID,
|
|
||||||
&result.AnalysisResult,
|
|
||||||
&result.MatchScore,
|
|
||||||
pq.Array(&result.KeySkills),
|
|
||||||
pq.Array(&result.MissingSkills),
|
|
||||||
&result.Recommendations,
|
|
||||||
&result.AnalyzedAt,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error scanning job analysis result: %v", err)
|
|
||||||
}
|
|
||||||
results = append(results, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return results, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobAnalysisResultRepository) GetAnalysisResultsByMatchScore(minScore int) ([]domain.JobAnalysisResult, error) {
|
|
||||||
query := `
|
|
||||||
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
|
|
||||||
FROM job_analysis_results
|
|
||||||
WHERE match_score >= $1
|
|
||||||
ORDER BY match_score DESC, analyzed_at DESC
|
|
||||||
`
|
|
||||||
|
|
||||||
rows, err := r.db.Query(query, minScore)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error querying job analysis results by match score: %v", err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var results []domain.JobAnalysisResult
|
|
||||||
for rows.Next() {
|
|
||||||
var result domain.JobAnalysisResult
|
|
||||||
err := rows.Scan(
|
|
||||||
&result.ID,
|
|
||||||
&result.JobID,
|
|
||||||
&result.AnalysisResult,
|
|
||||||
&result.MatchScore,
|
|
||||||
pq.Array(&result.KeySkills),
|
|
||||||
pq.Array(&result.MissingSkills),
|
|
||||||
&result.Recommendations,
|
|
||||||
&result.AnalyzedAt,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error scanning job analysis result: %v", err)
|
|
||||||
}
|
|
||||||
results = append(results, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return results, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
package repo
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/jobs-scraper/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
type JobDescriptionRepository struct {
|
|
||||||
db *sql.DB
|
|
||||||
}
|
|
||||||
|
|
||||||
type JobDescriptionData struct {
|
|
||||||
JobID int
|
|
||||||
Description string
|
|
||||||
Criteria map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewJobDescriptionRepository(db *sql.DB) *JobDescriptionRepository {
|
|
||||||
return &JobDescriptionRepository{db: db}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobDescriptionRepository) SaveJobDescriptions(jobDescriptions []domain.JobDescription) error {
|
|
||||||
if len(jobDescriptions) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build the VALUES clause dynamically
|
|
||||||
valueStrings := make([]string, 0, len(jobDescriptions))
|
|
||||||
valueArgs := make([]interface{}, 0, len(jobDescriptions)*3)
|
|
||||||
|
|
||||||
for i, jd := range jobDescriptions {
|
|
||||||
// Convert criteria map to JSONB
|
|
||||||
criteriaByte, err := json.Marshal(jd.Criteria)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error marshaling job criteria for job %d: %v", jd.JobID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i*3+1, i*3+2, i*3+3))
|
|
||||||
valueArgs = append(valueArgs, jd.JobID, jd.Description, criteriaByte)
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlStatement := fmt.Sprintf(`
|
|
||||||
INSERT INTO job_descriptions (job_id, description, job_criteria)
|
|
||||||
VALUES %s
|
|
||||||
ON CONFLICT (job_id) DO UPDATE SET
|
|
||||||
description = EXCLUDED.description,
|
|
||||||
job_criteria = EXCLUDED.job_criteria,
|
|
||||||
updated_at = CURRENT_TIMESTAMP
|
|
||||||
`, strings.Join(valueStrings, ","))
|
|
||||||
|
|
||||||
_, err := r.db.Exec(sqlStatement, valueArgs...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error saving job descriptions: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobDescriptionRepository) GetJobDescriptionByJobID(jobID int64) (string, map[string]string, error) {
|
|
||||||
var (
|
|
||||||
description string
|
|
||||||
criteriaByte []byte
|
|
||||||
criteria map[string]string
|
|
||||||
)
|
|
||||||
|
|
||||||
sqlStatement := `SELECT description, job_criteria FROM job_descriptions WHERE job_id = $1`
|
|
||||||
err := r.db.QueryRow(sqlStatement, jobID).Scan(&description, &criteriaByte)
|
|
||||||
if err != nil {
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
return "", nil, nil // No description found
|
|
||||||
}
|
|
||||||
return "", nil, fmt.Errorf("error fetching job description: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unmarshal the JSON criteria
|
|
||||||
if criteriaByte != nil {
|
|
||||||
if err := json.Unmarshal(criteriaByte, &criteria); err != nil {
|
|
||||||
return "", nil, fmt.Errorf("error unmarshaling job criteria: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return description, criteria, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,179 +0,0 @@
|
||||||
package repo
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/jobs-scraper/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
type JobRepository struct {
|
|
||||||
db *sql.DB
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewJobRepository(db *sql.DB) *JobRepository {
|
|
||||||
return &JobRepository{db: db}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobRepository) SaveJobs(jobs []domain.Job) error {
|
|
||||||
if len(jobs) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deduplicate jobs by ID to avoid duplicate errors
|
|
||||||
jobMap := make(map[int64]domain.Job)
|
|
||||||
for _, job := range jobs {
|
|
||||||
jobMap[job.ID] = job
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert to slice of pointers for the bulk function
|
|
||||||
uniqueJobs := make([]*domain.Job, 0, len(jobMap))
|
|
||||||
for _, job := range jobMap {
|
|
||||||
jobCopy := job // Important: create a copy to avoid pointer issues
|
|
||||||
uniqueJobs = append(uniqueJobs, &jobCopy)
|
|
||||||
}
|
|
||||||
|
|
||||||
sqlTemplate := `
|
|
||||||
INSERT INTO jobs (id, title, company, company_link, location, job_link, job_timestamp)
|
|
||||||
VALUES %s
|
|
||||||
ON CONFLICT (id) DO UPDATE SET
|
|
||||||
title = EXCLUDED.title,
|
|
||||||
company = EXCLUDED.company,
|
|
||||||
company_link = EXCLUDED.company_link,
|
|
||||||
location = EXCLUDED.location,
|
|
||||||
job_link = EXCLUDED.job_link,
|
|
||||||
job_timestamp = EXCLUDED.job_timestamp
|
|
||||||
`
|
|
||||||
|
|
||||||
sqlStatement, vals := prepareQueryCreateBulk(sqlTemplate, uniqueJobs)
|
|
||||||
|
|
||||||
_, err := r.db.Exec(sqlStatement, vals...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error inserting jobs: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobRepository) GetAllJobs() ([]domain.Job, error) {
|
|
||||||
rows, err := r.db.Query("SELECT * FROM jobs")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error querying jobs: %v", err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var jobs []domain.Job
|
|
||||||
for rows.Next() {
|
|
||||||
var job domain.Job
|
|
||||||
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil {
|
|
||||||
return nil, fmt.Errorf("error scanning job row: %v", err)
|
|
||||||
}
|
|
||||||
jobs = append(jobs, job)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, fmt.Errorf("error iterating over job rows: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobs, nil
|
|
||||||
}
|
|
||||||
func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|
||||||
var job domain.Job
|
|
||||||
|
|
||||||
sqlStatement := `
|
|
||||||
SELECT id, title, company, company_link, location, job_link, job_timestamp
|
|
||||||
FROM jobs
|
|
||||||
WHERE id = $1
|
|
||||||
`
|
|
||||||
|
|
||||||
err := r.db.QueryRow(sqlStatement, id).Scan(
|
|
||||||
&job.ID,
|
|
||||||
&job.Title,
|
|
||||||
&job.Company,
|
|
||||||
&job.CompanyLink,
|
|
||||||
&job.Location,
|
|
||||||
&job.JobLink,
|
|
||||||
&job.JobPostTime,
|
|
||||||
)
|
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
return nil, fmt.Errorf("job with ID %d not found", id)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error querying job: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &job, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
|
|
||||||
bf := bytes.Buffer{}
|
|
||||||
values := make([]interface{}, 0, len(models)*7)
|
|
||||||
|
|
||||||
for i, v := range models {
|
|
||||||
values = append(values, v.ID, v.Title, v.Company,
|
|
||||||
v.CompanyLink, v.Location, v.JobLink, v.JobPostTime,
|
|
||||||
)
|
|
||||||
|
|
||||||
numFields := 7 // the number of fields you are inserting
|
|
||||||
n := i * numFields
|
|
||||||
|
|
||||||
bf.WriteString("(")
|
|
||||||
for j := 0; j < numFields; j++ {
|
|
||||||
bf.WriteString("$")
|
|
||||||
bf.WriteString(strconv.Itoa(n + j + 1))
|
|
||||||
if j < numFields-1 {
|
|
||||||
bf.WriteString(", ")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
bf.WriteString(")")
|
|
||||||
if i < len(models)-1 {
|
|
||||||
bf.WriteString(", ")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf(s, bf.String()), values
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobRepository) GetJobsByStatus(status domain.JobStatus) ([]domain.Job, error) {
|
|
||||||
query := `
|
|
||||||
SELECT id, title, company, company_link, location, job_link, job_timestamp
|
|
||||||
FROM jobs
|
|
||||||
WHERE status = $1
|
|
||||||
`
|
|
||||||
|
|
||||||
rows, err := r.db.Query(query, int(status))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var jobs []domain.Job
|
|
||||||
for rows.Next() {
|
|
||||||
var job domain.Job
|
|
||||||
err := rows.Scan(
|
|
||||||
&job.ID,
|
|
||||||
&job.Title,
|
|
||||||
&job.Company,
|
|
||||||
&job.CompanyLink,
|
|
||||||
&job.Location,
|
|
||||||
&job.JobLink,
|
|
||||||
&job.JobPostTime,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
jobs = append(jobs, job)
|
|
||||||
}
|
|
||||||
|
|
||||||
return jobs, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *JobRepository) UpdateJobStatus(jobID int64, status domain.JobStatus) error {
|
|
||||||
query := `UPDATE jobs SET status = $1 WHERE id = $2`
|
|
||||||
_, err := r.db.Exec(query, int(status), jobID)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
// APIError represents an API error with status code and message
|
|
||||||
type APIError struct {
|
|
||||||
StatusCode int `json:"statusCode"`
|
|
||||||
Msg any `json:"msg"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error implements the error interface for APIError
|
|
||||||
func (e APIError) Error() string {
|
|
||||||
return fmt.Sprintf("api error: %d", e.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewAPIError creates a new APIError with the given status code and error message
|
|
||||||
func NewAPIError(statusCode int, err error) APIError {
|
|
||||||
return APIError{
|
|
||||||
StatusCode: statusCode,
|
|
||||||
Msg: err.Error(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// InvalidRequestData creates an APIError for validation errors with a map of field errors
|
|
||||||
func InvalidRequestData(msg string) APIError {
|
|
||||||
return APIError{
|
|
||||||
StatusCode: http.StatusUnprocessableEntity,
|
|
||||||
Msg: msg,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// InvalidJSON creates an APIError for invalid JSON format
|
|
||||||
func InvalidJSON() APIError {
|
|
||||||
return NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid JSON request data"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// APIResponse represents a standardized API response wrapper
|
|
||||||
type APIResponse struct {
|
|
||||||
Success bool `json:"success"`
|
|
||||||
Data any `json:"data,omitempty"`
|
|
||||||
Error any `json:"error,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteJSON writes the response as JSON with given status code, wrapped in a data object
|
|
||||||
func WriteJSON(w http.ResponseWriter, status int, data any) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(status)
|
|
||||||
|
|
||||||
response := APIResponse{
|
|
||||||
Success: status >= 200 && status < 300,
|
|
||||||
}
|
|
||||||
|
|
||||||
if response.Success {
|
|
||||||
response.Data = data
|
|
||||||
} else {
|
|
||||||
response.Error = data
|
|
||||||
}
|
|
||||||
|
|
||||||
json.NewEncoder(w).Encode(response)
|
|
||||||
}
|
|
||||||
|
|
||||||
// APIFunc is the signature for API handler functions
|
|
||||||
type APIFunc func(w http.ResponseWriter, r *http.Request) error
|
|
||||||
|
|
||||||
// Make wraps an APIFunc and handles errors consistently
|
|
||||||
func Make(h APIFunc) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if err := h(w, r); err != nil {
|
|
||||||
if apiErr, ok := err.(APIError); ok {
|
|
||||||
WriteJSON(w, apiErr.StatusCode, apiErr)
|
|
||||||
} else {
|
|
||||||
errResp := map[string]any{
|
|
||||||
"statusCode": http.StatusInternalServerError,
|
|
||||||
"msg": "internal server error",
|
|
||||||
}
|
|
||||||
WriteJSON(w, http.StatusInternalServerError, errResp)
|
|
||||||
// You could add logging here
|
|
||||||
slog.Error("HTTP API error", "err", err.Error(), "path", r.URL.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,425 +0,0 @@
|
||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
// "crypto/tls"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
// "net/url"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RetryConfig struct {
|
|
||||||
MaxRetries int
|
|
||||||
BaseDelay time.Duration
|
|
||||||
MaxDelay time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type RetryableHTTPRequestImpl struct {
|
|
||||||
client *http.Client
|
|
||||||
config RetryConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
|
|
||||||
return NewRetryableHTTPRequestWithProxy(config, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
|
|
||||||
// var transport *http.Transport
|
|
||||||
|
|
||||||
// if proxyURL != "" {
|
|
||||||
// proxyUrl, err := url.Parse(proxyURL)
|
|
||||||
// if err != nil {
|
|
||||||
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
|
|
||||||
// }
|
|
||||||
|
|
||||||
// transport = &http.Transport{
|
|
||||||
// Proxy: http.ProxyURL(proxyUrl),
|
|
||||||
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
client := &http.Client{
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
// Transport: transport,
|
|
||||||
}
|
|
||||||
|
|
||||||
return &RetryableHTTPRequestImpl{
|
|
||||||
client: client,
|
|
||||||
config: config,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
|
|
||||||
|
|
||||||
var lastErr error
|
|
||||||
|
|
||||||
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
|
|
||||||
req, err := http.NewRequestWithContext(ctx, method, url, body)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
|
||||||
for _, header := range headers {
|
|
||||||
req.Header.Set(header.Get("Key"), header.Get("Value"))
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := s.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
lastErr = err
|
|
||||||
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
|
|
||||||
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
|
|
||||||
// Success
|
|
||||||
return resp, nil
|
|
||||||
} else if resp.StatusCode == http.StatusTooManyRequests {
|
|
||||||
// 429 Too Many Requests - retry with backoff
|
|
||||||
resp.Body.Close()
|
|
||||||
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
|
|
||||||
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
|
|
||||||
} else {
|
|
||||||
// All other errors (4xx, 5xx) - don't retry
|
|
||||||
resp.Body.Close()
|
|
||||||
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// max attempts for now = 3
|
|
||||||
if attempt < s.config.MaxRetries {
|
|
||||||
// Increment delay by 2 seconds for each attempt
|
|
||||||
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
|
|
||||||
|
|
||||||
// Cap the delay to MaxDelay if set
|
|
||||||
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
|
|
||||||
delay = s.config.MaxDelay
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
case <-time.After(delay):
|
|
||||||
// Continue to next attempt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5.10.245.81:80
|
|
||||||
// 172.67.70.206:80
|
|
||||||
// 190.93.245.33:80
|
|
||||||
// 185.221.160.176:80
|
|
||||||
// 141.101.123.156:80
|
|
||||||
// 103.21.244.129:80
|
|
||||||
// 103.169.142.59:80
|
|
||||||
// 172.67.81.185:80
|
|
||||||
// 37.18.73.60:5566
|
|
||||||
// 209.38.83.56:1088
|
|
||||||
// 89.169.36.109:1080
|
|
||||||
// 103.21.244.248:80
|
|
||||||
// 172.67.70.92:80
|
|
||||||
// 172.67.88.34:80
|
|
||||||
// 141.101.120.38:80
|
|
||||||
// 172.67.75.255:80
|
|
||||||
// 172.67.75.171:80
|
|
||||||
// 45.131.4.157:80
|
|
||||||
// 141.101.121.208:80
|
|
||||||
// 141.101.120.70:80
|
|
||||||
// 103.21.244.70:80
|
|
||||||
// 103.21.244.140:80
|
|
||||||
// 103.21.244.138:80
|
|
||||||
// 154.16.146.43:80
|
|
||||||
// 128.199.202.122:3128
|
|
||||||
// 160.153.0.28:80
|
|
||||||
// 185.170.166.31:80
|
|
||||||
// 172.67.70.95:80
|
|
||||||
// 141.101.123.91:80
|
|
||||||
// 141.101.121.81:80
|
|
||||||
// 141.101.121.71:80
|
|
||||||
// 141.101.120.224:80
|
|
||||||
// 192.111.137.35:4145
|
|
||||||
// 45.12.31.20:80
|
|
||||||
// 103.21.244.150:80
|
|
||||||
// 103.21.244.214:80
|
|
||||||
// 103.21.244.29:80
|
|
||||||
// 195.85.23.88:80
|
|
||||||
// 23.227.39.134:80
|
|
||||||
// 66.42.224.229:41679
|
|
||||||
// 172.67.73.123:80
|
|
||||||
// 79.110.200.27:8000
|
|
||||||
// 5.182.34.33:80
|
|
||||||
// 103.21.244.174:80
|
|
||||||
// 172.67.75.203:80
|
|
||||||
// 141.101.122.150:80
|
|
||||||
// 141.101.120.35:80
|
|
||||||
// 175.47.237.95:6128
|
|
||||||
// 103.21.244.185:80
|
|
||||||
// 141.101.122.86:80
|
|
||||||
// 141.101.121.89:80
|
|
||||||
// 103.21.244.105:80
|
|
||||||
// 103.21.244.234:80
|
|
||||||
// 172.67.70.20:80
|
|
||||||
// 163.223.172.27:1080
|
|
||||||
// 172.67.84.128:80
|
|
||||||
// 103.21.244.57:80
|
|
||||||
// 31.43.179.191:80
|
|
||||||
// 173.245.59.61:80
|
|
||||||
// 141.101.120.48:80
|
|
||||||
// 185.162.230.19:80
|
|
||||||
// 172.67.172.150:80
|
|
||||||
// 159.112.235.63:80
|
|
||||||
// 172.64.40.184:80
|
|
||||||
// 103.21.244.55:80
|
|
||||||
// 103.21.244.49:80
|
|
||||||
// 172.67.70.228:80
|
|
||||||
// 103.21.244.8:80
|
|
||||||
// 66.235.200.77:80
|
|
||||||
// 45.85.119.147:80
|
|
||||||
// 172.67.177.231:80
|
|
||||||
// 173.245.49.28:80
|
|
||||||
// 110.232.92.49:8080
|
|
||||||
// 212.183.88.212:80
|
|
||||||
// 108.162.193.73:80
|
|
||||||
// 185.238.228.29:80
|
|
||||||
// 172.67.127.8:80
|
|
||||||
// 164.38.155.86:80
|
|
||||||
// 160.153.0.18:80
|
|
||||||
// 141.101.122.119:80
|
|
||||||
// 103.21.244.186:80
|
|
||||||
// 172.67.172.162:80
|
|
||||||
// 8.218.39.40:10800
|
|
||||||
// 47.237.132.101:60031
|
|
||||||
// 222.59.173.105:44193
|
|
||||||
// 143.110.190.60:1080
|
|
||||||
// 152.53.194.55:32059
|
|
||||||
// 115.187.50.40:5678
|
|
||||||
// 171.254.219.180:1080
|
|
||||||
// 103.165.157.155:8080
|
|
||||||
// 45.233.169.57:999
|
|
||||||
// 222.59.173.105:45035
|
|
||||||
// 202.40.179.18:4145
|
|
||||||
// 170.79.181.188:60606
|
|
||||||
// 37.186.66.36:3629
|
|
||||||
// 104.200.152.30:4145
|
|
||||||
// 106.13.58.110:8888
|
|
||||||
// 202.40.181.220:31247
|
|
||||||
// 43.135.36.240:80
|
|
||||||
// 103.82.27.107:10001
|
|
||||||
// 192.252.209.158:4145
|
|
||||||
// 142.54.236.97:4145
|
|
||||||
// 142.54.239.1:4145
|
|
||||||
// 184.170.245.148:4145
|
|
||||||
// 68.71.247.130:4145
|
|
||||||
// 72.195.34.58:4145
|
|
||||||
// 172.65.90.2:80
|
|
||||||
// 172.65.90.0:80
|
|
||||||
// 36.138.53.26:10019
|
|
||||||
// 183.215.23.242:9091
|
|
||||||
// 45.131.7.34:80
|
|
||||||
// 141.101.120.230:80
|
|
||||||
// 172.64.78.193:80
|
|
||||||
// 141.101.122.9:80
|
|
||||||
// 103.21.244.80:80
|
|
||||||
// 103.21.244.147:80
|
|
||||||
// 103.21.244.106:80
|
|
||||||
// 103.21.244.16:80
|
|
||||||
// 199.34.229.210:80
|
|
||||||
// 141.101.90.98:80
|
|
||||||
// 172.67.70.127:80
|
|
||||||
// 103.21.244.221:80
|
|
||||||
// 172.67.171.203:80
|
|
||||||
// 172.67.181.188:80
|
|
||||||
// 72.195.101.99:4145
|
|
||||||
// 141.101.120.106:80
|
|
||||||
// 103.21.244.180:80
|
|
||||||
// 103.21.244.102:80
|
|
||||||
// 172.67.70.233:80
|
|
||||||
// 108.162.193.107:80
|
|
||||||
// 45.131.208.112:80
|
|
||||||
// 154.194.12.195:80
|
|
||||||
// 69.84.182.23:80
|
|
||||||
// 141.101.120.128:80
|
|
||||||
// 141.101.120.149:80
|
|
||||||
// 141.101.120.232:80
|
|
||||||
// 141.101.120.6:80
|
|
||||||
// 172.67.98.18:80
|
|
||||||
// 141.101.113.20:80
|
|
||||||
// 208.65.90.21:4145
|
|
||||||
// 198.177.254.131:4145
|
|
||||||
// 142.54.237.38:4145
|
|
||||||
// 192.252.220.89:4145
|
|
||||||
// 199.187.210.54:4145
|
|
||||||
// 199.102.105.242:4145
|
|
||||||
// 199.102.107.145:4145
|
|
||||||
// 198.8.84.3:4145
|
|
||||||
// 98.182.171.161:4145
|
|
||||||
// 192.111.129.150:4145
|
|
||||||
// 184.170.248.5:4145
|
|
||||||
// 98.188.47.150:4145
|
|
||||||
// 125.228.94.232:4145
|
|
||||||
// 125.228.94.153:4145
|
|
||||||
// 185.162.229.219:80
|
|
||||||
// 103.21.244.22:80
|
|
||||||
// 172.64.89.97:80
|
|
||||||
// 103.21.244.54:80
|
|
||||||
// 103.21.244.37:80
|
|
||||||
// 141.101.123.225:80
|
|
||||||
// 103.21.244.168:80
|
|
||||||
// 103.21.244.149:80
|
|
||||||
// 103.21.244.100:80
|
|
||||||
// 103.21.244.24:80
|
|
||||||
// 23.227.39.209:80
|
|
||||||
// 172.67.83.15:80
|
|
||||||
// 45.131.4.241:80
|
|
||||||
// 141.101.120.201:80
|
|
||||||
// 172.67.191.237:80
|
|
||||||
// 172.67.127.188:80
|
|
||||||
// 103.21.244.92:80
|
|
||||||
// 103.160.204.22:80
|
|
||||||
// 220.197.44.36:3128
|
|
||||||
// 39.185.41.193:5911
|
|
||||||
// 23.227.39.121:80
|
|
||||||
// 139.162.78.109:80
|
|
||||||
// 188.114.99.144:80
|
|
||||||
// 45.131.5.37:80
|
|
||||||
// 45.131.4.250:80
|
|
||||||
// 23.227.39.65:80
|
|
||||||
// 141.101.121.191:80
|
|
||||||
// 172.67.188.16:80
|
|
||||||
// 172.67.254.148:80
|
|
||||||
// 221.1.104.177:7302
|
|
||||||
// 222.59.173.105:44008
|
|
||||||
// 103.21.244.83:80
|
|
||||||
// 172.64.149.1:80
|
|
||||||
// 45.12.30.22:80
|
|
||||||
// 222.59.173.105:44027
|
|
||||||
// 172.64.149.26:80
|
|
||||||
// 103.21.244.46:80
|
|
||||||
// 45.12.31.242:80
|
|
||||||
// 23.227.38.195:80
|
|
||||||
// 172.67.177.162:80
|
|
||||||
// 103.160.204.200:80
|
|
||||||
// 172.67.70.129:80
|
|
||||||
// 141.101.123.245:80
|
|
||||||
// 185.162.230.117:80
|
|
||||||
// 185.162.230.183:80
|
|
||||||
// 69.61.200.104:36181
|
|
||||||
// 208.65.90.3:4145
|
|
||||||
// 72.223.188.92:4145
|
|
||||||
// 192.252.214.17:4145
|
|
||||||
// 68.71.242.118:4145
|
|
||||||
// 192.252.210.233:4145
|
|
||||||
// 107.181.161.81:4145
|
|
||||||
// 107.181.168.145:4145
|
|
||||||
// 206.220.175.2:4145
|
|
||||||
// 72.37.217.3:4145
|
|
||||||
// 192.252.208.70:14282
|
|
||||||
// 70.166.167.55:57745
|
|
||||||
// 192.111.137.37:18762
|
|
||||||
// 98.188.47.132:4145
|
|
||||||
// 62.99.138.162:80
|
|
||||||
// 172.64.90.186:80
|
|
||||||
// 172.67.74.57:80
|
|
||||||
// 45.131.6.67:80
|
|
||||||
// 141.101.122.37:80
|
|
||||||
// 103.21.244.134:80
|
|
||||||
// 172.64.155.71:80
|
|
||||||
// 5.182.34.139:80
|
|
||||||
// 45.131.6.31:80
|
|
||||||
// 141.101.121.21:80
|
|
||||||
// 141.101.120.190:80
|
|
||||||
// 103.21.244.161:80
|
|
||||||
// 103.21.244.156:80
|
|
||||||
// 103.21.244.151:80
|
|
||||||
// 172.64.89.0:80
|
|
||||||
// 58.216.109.17:800
|
|
||||||
// 58.241.88.18:800
|
|
||||||
// 36.147.78.166:80
|
|
||||||
// 170.244.26.36:8888
|
|
||||||
// 170.244.25.52:8888
|
|
||||||
// 170.244.26.206:8888
|
|
||||||
// 36.138.53.26:10017
|
|
||||||
// 201.148.32.162:80
|
|
||||||
// 203.19.38.114:1080
|
|
||||||
// 31.43.179.204:80
|
|
||||||
// 45.12.31.50:80
|
|
||||||
// 170.244.27.142:8888
|
|
||||||
// 170.244.26.195:8888
|
|
||||||
// 170.244.27.58:8888
|
|
||||||
// 170.244.27.61:8888
|
|
||||||
// 170.244.27.150:8888
|
|
||||||
// 47.243.94.125:1080
|
|
||||||
// 40.177.65.8:80
|
|
||||||
// 47.57.13.107:80
|
|
||||||
// 194.158.203.14:80
|
|
||||||
// 194.219.134.234:80
|
|
||||||
// 103.21.244.51:80
|
|
||||||
// 103.21.244.160:80
|
|
||||||
// 103.21.244.144:80
|
|
||||||
// 213.33.126.130:80
|
|
||||||
// 103.21.244.69:80
|
|
||||||
// 103.21.244.192:80
|
|
||||||
// 103.21.244.133:80
|
|
||||||
// 141.193.213.189:80
|
|
||||||
// 68.71.254.6:4145
|
|
||||||
// 185.221.160.21:80
|
|
||||||
// 188.114.99.97:80
|
|
||||||
// 63.141.128.94:80
|
|
||||||
// 141.193.213.213:80
|
|
||||||
// 185.162.228.234:80
|
|
||||||
// 213.143.113.82:80
|
|
||||||
// 154.194.12.207:80
|
|
||||||
// 189.203.181.34:1080
|
|
||||||
// 202.144.134.150:5678
|
|
||||||
// 190.242.157.215:8080
|
|
||||||
// 72.49.49.11:31034
|
|
||||||
// 142.54.237.34:4145
|
|
||||||
// 68.71.249.153:48606
|
|
||||||
// 192.252.209.155:14455
|
|
||||||
// 192.252.216.86:4145
|
|
||||||
// 142.54.229.249:4145
|
|
||||||
// 209.97.150.167:8080
|
|
||||||
// 192.252.220.92:17328
|
|
||||||
// 98.178.72.21:10919
|
|
||||||
// 144.124.228.87:1080
|
|
||||||
// 183.240.46.42:80
|
|
||||||
// 32.223.6.94:80
|
|
||||||
// 192.252.214.20:15864
|
|
||||||
// 198.177.252.24:4145
|
|
||||||
// 192.252.208.67:14287
|
|
||||||
// 198.8.94.170:4145
|
|
||||||
// 68.71.241.33:4145
|
|
||||||
// 98.181.137.83:4145
|
|
||||||
// 45.12.30.139:80
|
|
||||||
// 103.21.244.97:80
|
|
||||||
// 65.1.148.157:80
|
|
||||||
// 199.58.184.97:4145
|
|
||||||
// 141.101.120.148:80
|
|
||||||
// 185.238.228.77:80
|
|
||||||
// 172.67.202.134:80
|
|
||||||
// 172.67.182.49:80
|
|
||||||
// 45.131.210.112:80
|
|
||||||
// 141.101.120.100:80
|
|
||||||
// 103.21.244.59:80
|
|
||||||
// 103.21.244.189:80
|
|
||||||
// 103.21.244.154:80
|
|
||||||
// 103.21.244.23:80
|
|
||||||
// 173.245.49.40:80
|
|
||||||
// 82.200.235.134:38191
|
|
||||||
// 43.224.118.89:2626
|
|
||||||
// 119.148.47.226:16464
|
|
||||||
// 211.230.49.122:3128
|
|
||||||
// 62.171.159.232:8888
|
|
||||||
// 115.127.112.34:1080
|
|
||||||
// 185.191.236.162:3128
|
|
||||||
// 41.223.119.156:3128
|
|
||||||
// 103.138.123.242:8082
|
|
||||||
// 121.169.46.116:1090
|
|
||||||
// 185.145.185.218:8080
|
|
||||||
// 45.166.93.113:999
|
|
||||||
// 163.53.204.178:9813
|
|
||||||
// 181.224.226.154:8080
|
|
||||||
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/eduardolat/openroutergo"
|
"github.com/eduardolat/openroutergo"
|
||||||
"github.com/jobs-scraper/internal/domain"
|
"github.com/jobs-scraper/shared/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
// JobAnalysisResult represents the structured response from job analysis
|
// JobAnalysisResult represents the structured response from job analysis
|
||||||
7
shared/server/go.mod
Normal file
7
shared/server/go.mod
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
module github.com/jobs-scraper/shared/server
|
||||||
|
|
||||||
|
go 1.24.0
|
||||||
|
|
||||||
|
require github.com/gorilla/mux v1.8.1
|
||||||
|
|
||||||
|
replace github.com/jobs-scraper/shared/server => ./
|
||||||
2
shared/server/go.sum
Normal file
2
shared/server/go.sum
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||||
|
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||||
Loading…
Reference in a new issue