diff --git a/api/app/app.go b/api/app/app.go index 73d6898..c19fdc1 100644 --- a/api/app/app.go +++ b/api/app/app.go @@ -5,10 +5,10 @@ import ( "github.com/gorilla/mux" "github.com/jobs-scraper/api/commands/job" - "github.com/jobs-scraper/infrastructure/rabbitmq" - "github.com/jobs-scraper/internal/ports" - "github.com/jobs-scraper/internal/repo" - "github.com/jobs-scraper/internal/server" + "github.com/jobs-scraper/shared/infrastructure/rabbitmq" + "github.com/jobs-scraper/shared/ports" + "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/shared/server" ) type Application struct { diff --git a/api/commands/job/create-job.go b/api/commands/job/create-job.go index a0d1ce4..1b6dc2f 100644 --- a/api/commands/job/create-job.go +++ b/api/commands/job/create-job.go @@ -5,10 +5,10 @@ import ( "errors" "log" - "github.com/jobs-scraper/infrastructure/rabbitmq" - "github.com/jobs-scraper/internal/domain" - "github.com/jobs-scraper/internal/ports" - "github.com/jobs-scraper/internal/repo" + "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/shared/infrastructure/rabbitmq" + "github.com/jobs-scraper/shared/ports" + "github.com/jobs-scraper/shared/repo" ) type CreateJob struct { diff --git a/api/main.go b/api/main.go index 9b0aa57..d255101 100644 --- a/api/main.go +++ b/api/main.go @@ -11,11 +11,11 @@ import ( "github.com/gorilla/mux" "github.com/jobs-scraper/api/app" - _ "github.com/jobs-scraper/docs" - "github.com/jobs-scraper/infrastructure" - httpHandler "github.com/jobs-scraper/infrastructure/http" - "github.com/jobs-scraper/infrastructure/rabbitmq" - "github.com/jobs-scraper/internal/repo" + "github.com/jobs-scraper/shared/infrastructure" + httpHandler "github.com/jobs-scraper/shared/infrastructure/http" + "github.com/jobs-scraper/shared/infrastructure/rabbitmq" + "github.com/jobs-scraper/shared/repo" + _ "github.com/jobs-scraper/swagger" "github.com/joho/godotenv" httpSwagger "github.com/swaggo/http-swagger" ) @@ -72,7 +72,7 @@ func main() { // Create job analysis result repository jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) - + jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo) jobHandler.RegisterRoutes(router) diff --git a/cmd/cron-analyzer/main.go b/cmd/cron-analyzer/main.go index 37e1db9..b50f1cb 100644 --- a/cmd/cron-analyzer/main.go +++ b/cmd/cron-analyzer/main.go @@ -4,7 +4,7 @@ import ( "log" "github.com/jobs-scraper/cron" - "github.com/jobs-scraper/infrastructure" + "github.com/jobs-scraper/shared/infrastructure" "github.com/joho/godotenv" ) @@ -32,7 +32,7 @@ func main() { log.Println("Successfully connected to db") analyzer := cron.NewJobAnalyzer(db) - + if err := analyzer.AnalyzeJobs(); err != nil { log.Fatalf("Error analyzing jobs: %v", err) } diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 1fa097f..775fe8d 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/jobs-scraper/infrastructure" + "github.com/jobs-scraper/shared/infrastructure" "github.com/joho/godotenv" ) diff --git a/cron/analyze-jobs.go b/cron/analyze-jobs.go index d87e262..1b6d1dd 100644 --- a/cron/analyze-jobs.go +++ b/cron/analyze-jobs.go @@ -6,19 +6,19 @@ import ( "log" "os" - "github.com/jobs-scraper/infrastructure/rabbitmq" - "github.com/jobs-scraper/internal/domain" - "github.com/jobs-scraper/internal/repo" - "github.com/jobs-scraper/internal/services" + "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/shared/infrastructure/rabbitmq" + "github.com/jobs-scraper/shared/openrouter" + "github.com/jobs-scraper/shared/repo" ) type JobAnalyzer struct { - db *sql.DB - jobRepo *repo.JobRepository - jobDescriptionRepo *repo.JobDescriptionRepository - jobAnalysisResultRepo *repo.JobAnalysisResultRepository - openRouterService services.OpenRouterService - rmq *rabbitmq.RabbitMQClient + db *sql.DB + jobRepo *repo.JobRepository + jobDescriptionRepo *repo.JobDescriptionRepository + jobAnalysisResultRepo *repo.JobAnalysisResultRepository + openRouterService services.OpenRouterService + rmq *rabbitmq.RabbitMQClient } func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { @@ -122,7 +122,7 @@ func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error { message := rabbitmq.CvAnalyzeMessage{ JobID: job.ID, } - + if err := ja.publishCvAnalysisMessage(message); err != nil { log.Printf("Warning: Failed to publish CV analysis message: %v", err) } diff --git a/cv/main.go b/cv/main.go index e7129bd..81b81b7 100644 --- a/cv/main.go +++ b/cv/main.go @@ -4,10 +4,10 @@ import ( "log" "os" - "github.com/jobs-scraper/infrastructure" - "github.com/jobs-scraper/internal/domain" - "github.com/jobs-scraper/internal/repo" - "github.com/jobs-scraper/internal/services" + "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/shared/infrastructure" + "github.com/jobs-scraper/shared/openrouter" + "github.com/jobs-scraper/shared/repo" "github.com/joho/godotenv" ) diff --git a/go.work b/go.work index 06b2b83..b7e725a 100644 --- a/go.work +++ b/go.work @@ -7,5 +7,6 @@ use ( ./shared/infrastructure ./shared/ports ./shared/repo + ./shared/server ./shared/utils ) diff --git a/infrastructure/db.go b/infrastructure/db.go deleted file mode 100644 index 7657b91..0000000 --- a/infrastructure/db.go +++ /dev/null @@ -1,171 +0,0 @@ -package infrastructure - -import ( - "database/sql" - "errors" - "fmt" - "os" - "path/filepath" - "runtime" - - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/golang-migrate/migrate/v4/source/file" - _ "github.com/lib/pq" -) - -// Config holds database configuration -type Config struct { - Host string - Port string - User string - Password string - DBName string - SSLMode string -} - -// NewConnection creates a new database connection -func NewConnection(config Config) (*sql.DB, error) { - dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", - config.Host, config.Port, config.User, config.Password, config.DBName, config.SSLMode) - - db, err := sql.Open("postgres", dsn) - if err != nil { - return nil, fmt.Errorf("failed to open database connection: %w", err) - } - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping database: %w", err) - } - - return db, nil -} - -// LoadConfigFromEnv loads database configuration from environment variables -func LoadConfigFromEnv() Config { - return Config{ - Host: getEnv("DB_HOST", "localhost"), - Port: getEnv("DB_PORT", "5432"), - User: getEnv("DB_USER", "postgres"), - Password: getEnv("DB_PASSWORD", "password"), - DBName: getEnv("DB_NAME", "linkedin_jobs"), - SSLMode: getEnv("DB_SSLMODE", "disable"), - } -} - -func getEnv(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} - -// RunMigrations runs all pending migrations -func RunMigrations(db *sql.DB) error { - driver, err := postgres.WithInstance(db, &postgres.Config{}) - if err != nil { - return fmt.Errorf("failed to create postgres driver: %w", err) - } - - // Get the migrations directory path - migrationsPath, err := getMigrationsPath() - if err != nil { - return fmt.Errorf("failed to get migrations path: %w", err) - } - - m, err := migrate.NewWithDatabaseInstance( - fmt.Sprintf("file://%s", migrationsPath), - "postgres", - driver, - ) - if err != nil { - return fmt.Errorf("failed to create migrate instance: %w", err) - } - - if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) { - return fmt.Errorf("failed to run migrations: %w", err) - } - - return nil -} - -// RollbackMigrations rolls back migrations by the specified number of steps -func RollbackMigrations(db *sql.DB, steps int) error { - driver, err := postgres.WithInstance(db, &postgres.Config{}) - if err != nil { - return fmt.Errorf("failed to create postgres driver: %w", err) - } - - migrationsPath, err := getMigrationsPath() - if err != nil { - return fmt.Errorf("failed to get migrations path: %w", err) - } - - m, err := migrate.NewWithDatabaseInstance( - fmt.Sprintf("file://%s", migrationsPath), - "postgres", - driver, - ) - if err != nil { - return fmt.Errorf("failed to create migrate instance: %w", err) - } - defer m.Close() - - if err := m.Steps(-steps); err != nil && !errors.Is(err, migrate.ErrNoChange) { - return fmt.Errorf("failed to rollback migrations: %w", err) - } - - return nil -} - -// GetMigrationVersion returns the current migration version -func GetMigrationVersion(db *sql.DB) (uint, bool, error) { - driver, err := postgres.WithInstance(db, &postgres.Config{}) - if err != nil { - return 0, false, fmt.Errorf("failed to create postgres driver: %w", err) - } - - migrationsPath, err := getMigrationsPath() - if err != nil { - return 0, false, fmt.Errorf("failed to get migrations path: %w", err) - } - - m, err := migrate.NewWithDatabaseInstance( - fmt.Sprintf("file://%s", migrationsPath), - "postgres", - driver, - ) - if err != nil { - return 0, false, fmt.Errorf("failed to create migrate instance: %w", err) - } - defer m.Close() - - version, dirty, err := m.Version() - if err != nil { - if errors.Is(err, migrate.ErrNilVersion) { - return 0, false, nil - } - return 0, false, fmt.Errorf("failed to get migration version: %w", err) - } - - return version, dirty, nil -} - -func getMigrationsPath() (string, error) { - // Get the directory of the current source file - _, filename, _, ok := runtime.Caller(0) - if !ok { - return "", fmt.Errorf("failed to get current file path") - } - - // Get the project root (go up from infrastructure/ to project root) - projectRoot := filepath.Dir(filepath.Dir(filename)) - migrationsPath := filepath.Join(projectRoot, "migrations") - - // Verify the migrations directory exists - if _, err := os.Stat(migrationsPath); err != nil { - return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err) - } - - return migrationsPath, nil -} diff --git a/infrastructure/http/job.go b/infrastructure/http/job.go deleted file mode 100644 index 23daa8d..0000000 --- a/infrastructure/http/job.go +++ /dev/null @@ -1,148 +0,0 @@ -package http - -import ( - "encoding/json" - "fmt" - "log/slog" - "net/http" - "strconv" - - "github.com/jobs-scraper/internal/ports" - "github.com/jobs-scraper/internal/repo" - "github.com/jobs-scraper/internal/utils" - - "github.com/gorilla/mux" -) - -// JobHandler handles HTTP requests related to jobs -type JobHandler struct { - jobCommands *ports.JobCommands - jobAnalysisResultRepo *repo.JobAnalysisResultRepository -} - -// NewJobHandler creates a new instance of JobHandler -func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler { - return &JobHandler{ - jobCommands: jobCommands, - jobAnalysisResultRepo: jobAnalysisResultRepo, - } -} - -// RegisterRoutes registers all routes to the router -func (h *JobHandler) RegisterRoutes(router *mux.Router) { - jobs := router.PathPrefix("/jobs").Subrouter() - - //jobs.Use(AuthMiddleware) - - jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST") - jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET") - jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") - jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") -} - -// CreateJob handles the creation of a new job -// @Summary Create a new job -// @Description Creates a new job with the provided specifications -// @Tags jobs -// @Accept json -// @Produce json -// @Param job body ports.CreateJobCommand true "Job creation data" -// @Success 201 {object} map[string]string "Successfully created job" -// @Failure 400 {object} map[string]string "Invalid request data" -// @Failure 500 {object} map[string]string "Internal server error" -// @Router /jobs [post] -func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { - var cmd ports.CreateJobCommand - if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil { - slog.Error(err.Error()) - return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON()) - } - - err := h.jobCommands.CreateJob.Handle(cmd) - if err != nil { - slog.Error(err.Error()) - return utils.NewAPIError(http.StatusInternalServerError, err) - } - - utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"}) - return nil -} - -// GetAllAnalysisResults handles retrieving all job analysis results -// @Summary Get all job analysis results -// @Description Retrieves all job analysis results from the database -// @Tags jobs -// @Produce json -// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results" -// @Failure 500 {object} map[string]string "Internal server error" -// @Router /jobs/analysis [get] -func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error { - results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults() - if err != nil { - slog.Error("Failed to get analysis results", "error", err) - return utils.NewAPIError(http.StatusInternalServerError, err) - } - - utils.WriteJSON(w, http.StatusOK, results) - return nil -} - -// GetJobAnalysisResult handles retrieving analysis result for a specific job -// @Summary Get job analysis result by job ID -// @Description Retrieves the analysis result for a specific job -// @Tags jobs -// @Produce json -// @Param id path int true "Job ID" -// @Success 200 {object} domain.JobAnalysisResult "Job analysis result" -// @Failure 400 {object} map[string]string "Invalid job ID" -// @Failure 404 {object} map[string]string "Analysis result not found" -// @Failure 500 {object} map[string]string "Internal server error" -// @Router /jobs/{id}/analysis [get] -func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error { - vars := mux.Vars(r) - jobID, err := strconv.ParseInt(vars["id"], 10, 64) - if err != nil { - return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID")) - } - - result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID) - if err != nil { - slog.Error("Failed to get analysis result", "jobID", jobID, "error", err) - return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found")) - } - - utils.WriteJSON(w, http.StatusOK, result) - return nil -} - -// GetTopMatches handles retrieving top matching jobs based on analysis score -// @Summary Get top matching jobs -// @Description Retrieves jobs with high analysis match scores -// @Tags jobs -// @Produce json -// @Param min_score query int false "Minimum match score (default: 70)" -// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results" -// @Failure 400 {object} map[string]string "Invalid minimum score" -// @Failure 500 {object} map[string]string "Internal server error" -// @Router /jobs/analysis/top-matches [get] -func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error { - minScoreStr := r.URL.Query().Get("min_score") - minScore := 70 // default minimum score - - if minScoreStr != "" { - var err error - minScore, err = strconv.Atoi(minScoreStr) - if err != nil { - return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score")) - } - } - - results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore) - if err != nil { - slog.Error("Failed to get top matches", "minScore", minScore, "error", err) - return utils.NewAPIError(http.StatusInternalServerError, err) - } - - utils.WriteJSON(w, http.StatusOK, results) - return nil -} diff --git a/infrastructure/http/middlewares.go b/infrastructure/http/middlewares.go deleted file mode 100644 index 3b6e0da..0000000 --- a/infrastructure/http/middlewares.go +++ /dev/null @@ -1,143 +0,0 @@ -package http - -import ( - "context" - "encoding/json" - "log" - "net/http" - "strings" - "time" -) - -// CORSMiddleware handles Cross-Origin Resource Sharing -func CORSMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Set CORS headers - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") - w.Header().Set("Access-Control-Allow-Credentials", "true") - w.Header().Set("Access-Control-Max-Age", "86400") - - // Handle preflight OPTIONS requests - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusOK) - return - } - - next.ServeHTTP(w, r) - }) -} - -// LogsMiddleware logs HTTP requests with timing information -func LogsMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - next.ServeHTTP(w, r) - log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start)) - }) -} - -// AuthMiddleware handles JWT token validation, API keys, and basic auth -func AuthMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Skip auth for swagger endpoints and health checks - if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" { - next.ServeHTTP(w, r) - return - } - - // Try Bearer token authentication first - authHeader := r.Header.Get("Authorization") - if authHeader != "" { - if strings.HasPrefix(authHeader, "Bearer ") { - token := strings.TrimPrefix(authHeader, "Bearer ") - if token != "" { - userID, err := validateJWTToken(token) - if err == nil { - ctx := context.WithValue(r.Context(), "user_id", userID) - next.ServeHTTP(w, r.WithContext(ctx)) - return - } - } - } - } - - // Try API Key authentication - apiKey := r.Header.Get("X-API-Key") - if apiKey != "" { - if isValidAPIKey(apiKey) { - ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey)) - next.ServeHTTP(w, r.WithContext(ctx)) - return - } - } - - // Try Basic authentication - username, password, ok := r.BasicAuth() - if ok { - if isValidCredentials(username, password) { - ctx := context.WithValue(r.Context(), "username", username) - next.ServeHTTP(w, r.WithContext(ctx)) - return - } - } - - // Fallback to cookie-based auth (existing behavior) - token, err := r.Cookie("token") - if err == nil && token.Value == "123" { - next.ServeHTTP(w, r) - return - } - - // No valid authentication found - response := map[string]interface{}{ - "error": "Unauthorized", - "message": "Valid authentication required. Use Bearer token, API key, or basic auth.", - "code": http.StatusUnauthorized, - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusUnauthorized) - json.NewEncoder(w).Encode(response) - }) -} - -// Helper functions for authentication validation -func validateJWTToken(token string) (string, error) { - // TODO: Implement JWT validation logic with your preferred JWT library - // For demonstration, accept a demo token - if token == "valid-jwt-token" { - return "user-123", nil - } - return "", http.ErrAbortHandler -} - -func isValidAPIKey(apiKey string) bool { - // TODO: Implement API key validation logic - // For demonstration, accept a demo API key - return apiKey == "demo-api-key-123" -} - -func getUserIDFromAPIKey(apiKey string) string { - // TODO: Implement logic to get user ID from API key - // For demonstration, return a mock user ID - return "api-user-123" -} - -func isValidCredentials(username, password string) bool { - // TODO: Implement credential validation logic - // For demonstration, accept demo credentials - return username == "admin" && password == "password" -} - -// GetUserIDFromContext extracts user ID from request context -func GetUserIDFromContext(ctx context.Context) (string, bool) { - userID, ok := ctx.Value("user_id").(string) - return userID, ok -} - -// GetUsernameFromContext extracts username from request context -func GetUsernameFromContext(ctx context.Context) (string, bool) { - username, ok := ctx.Value("username").(string) - return username, ok -} diff --git a/infrastructure/rabbitmq/messages.go b/infrastructure/rabbitmq/messages.go deleted file mode 100644 index a9d3c52..0000000 --- a/infrastructure/rabbitmq/messages.go +++ /dev/null @@ -1,5 +0,0 @@ -package rabbitmq - -type CvAnalyzeMessage struct { - JobID int64 `json:"jobID"` -} diff --git a/infrastructure/rabbitmq/rabbitmq.go b/infrastructure/rabbitmq/rabbitmq.go deleted file mode 100644 index 4610e3d..0000000 --- a/infrastructure/rabbitmq/rabbitmq.go +++ /dev/null @@ -1,248 +0,0 @@ -package rabbitmq - -import ( - "fmt" - "log" - "os" - "time" - - amqp "github.com/rabbitmq/amqp091-go" -) - -const ( - ScraperExchange = "scraper_exchange" - LinkedInQueue = "scraper.linkedin" - IndeedQueue = "scraper.indeed" - BaytQueue = "scraper.bayt" - TokyoDevQueue = "scraper.tokyodev" - GlassDoorQueue = "scraper.glassdoor" - JapanDevQueue = "scraper.japandev" - DeadLetterExchange = "scraper_dlx" - CvAnalyzeExchange = "cv_exchange" - CvAnalyzeQueue = "cv.analyze" -) - -type RabbitMQClient struct { - Conn *amqp.Connection - Channel *amqp.Channel -} - -type MessageHandler func([]byte) error - -func NewRabbitMQClient() (*RabbitMQClient, error) { - // Get RabbitMQ URL from environment or use default - rabbitmqURL := os.Getenv("RABBITMQ_URL") - if rabbitmqURL == "" { - rabbitmqURL = "amqp://guest:guest@localhost:5672/" - } - - // Connect to RabbitMQ with retry logic - var conn *amqp.Connection - var err error - - for i := range 5 { - conn, err = amqp.Dial(rabbitmqURL) - if err == nil { - break - } - log.Printf("Failed to connect to RabbitMQ (attempt %d/5): %v", i+1, err) - time.Sleep(5 * time.Second) - } - - if err != nil { - return nil, fmt.Errorf("error connecting to RabbitMQ after 5 attempts: %v", err) - } - - // Create a channel - ch, err := conn.Channel() - if err != nil { - conn.Close() - return nil, fmt.Errorf("error creating channel: %v", err) - } - - client := &RabbitMQClient{ - Conn: conn, - Channel: ch, - } - - // Setup exchanges and queues - if err := client.setupInfrastructure(); err != nil { - client.Close() - return nil, fmt.Errorf("error setting up infrastructure: %v", err) - } - - log.Println("Successfully connected to RabbitMQ and set up infrastructure") - return client, nil -} - -func (r *RabbitMQClient) setupInfrastructure() error { - // Declare dead letter exchange - err := r.Channel.ExchangeDeclare( - DeadLetterExchange, - "direct", - true, // durable - false, // auto-delete - false, // internal - false, // no-wait - nil, // arguments - ) - if err != nil { - return fmt.Errorf("error declaring dead letter exchange: %v", err) - } - - // Declare main exchange - err = r.Channel.ExchangeDeclare( - ScraperExchange, - "topic", - true, // durable - false, // auto-delete - false, // internal - false, // no-wait - nil, // arguments - ) - if err != nil { - return fmt.Errorf("error declaring exchange: %v", err) - } - - // Define queues with their routing keys - queues := map[string]string{ - LinkedInQueue: "scraper.linkedin", - IndeedQueue: "scraper.indeed", - BaytQueue: "scraper.bayt", - TokyoDevQueue: "scraper.tokyodev", - JapanDevQueue: "scraper.japandev", - GlassDoorQueue: "scraper.glassdoor", - } - - // Declare queues with dead letter exchange and TTL - for queueName, routingKey := range queues { - dlqName := queueName + "_dlq" - _, err := r.Channel.QueueDeclare( - dlqName, - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait - nil, // arguments - ) - if err != nil { - return fmt.Errorf("error declaring dead letter queue %s: %v", dlqName, err) - } - - err = r.Channel.QueueBind( - dlqName, - queueName, // routing key is the original queue name - DeadLetterExchange, - false, - nil, - ) - if err != nil { - return fmt.Errorf("error binding dead letter queue %s: %v", dlqName, err) - } - - _, err = r.Channel.QueueDeclare( - queueName, - true, // durable - false, // delete when unused - false, // exclusive - false, // no-wait - amqp.Table{ - "x-dead-letter-exchange": DeadLetterExchange, - "x-dead-letter-routing-key": queueName, - "x-message-ttl": int64(24 * time.Hour / time.Millisecond), // 24 hours TTL - "x-max-retries": 3, - }, - ) - if err != nil { - return fmt.Errorf("error declaring queue %s: %v", queueName, err) - } - - err = r.Channel.QueueBind( - queueName, - routingKey, - ScraperExchange, - false, - nil, - ) - if err != nil { - return fmt.Errorf("error binding queue %s: %v", queueName, err) - } - } - - return nil -} - -// Publish publishes a message to a specific routing key -func (r *RabbitMQClient) Publish(routingKey, exchange string, data []byte) error { - log.Printf("Publishing to exchange=%s, routingKey=%s, data=%s", ScraperExchange, routingKey, string(data)) - err := r.Channel.Publish( - ScraperExchange, - routingKey, - false, - false, - amqp.Publishing{ - ContentType: "application/json", - Body: data, - DeliveryMode: amqp.Persistent, - Timestamp: time.Now().UTC(), - }, - ) - if err != nil { - log.Printf("Error publishing: %v", err) - return fmt.Errorf("error publishing message: %v", err) - } - return nil -} - -// Subscribe creates a subscription to a specific queue with production-ready error handling -func (r *RabbitMQClient) Subscribe(queueName, consumerName string, handler MessageHandler) error { - err := r.Channel.Qos( - 1, // prefetch count - 0, // prefetch size - false, // global - ) - if err != nil { - return fmt.Errorf("error setting QoS: %v", err) - } - - // Start consuming messages - msgs, err := r.Channel.Consume( - queueName, - consumerName, - false, // auto-ack - false, // exclusive - false, // no-local - false, // no-wait - nil, // args - ) - if err != nil { - return fmt.Errorf("error starting consumer: %v", err) - } - - go func() { - for msg := range msgs { - // Call the handler - err := handler(msg.Body) - if err != nil { - log.Printf("Error processing message: %v", err) - // Reject and requeue the message (will go to DLX after max retries) - msg.Nack(false, true) - } else { - msg.Ack(false) - } - } - }() - - log.Printf("Successfully subscribed to queue: %s ", queueName) - return nil -} - -// Close closes the RabbitMQ connection and channel -func (r *RabbitMQClient) Close() { - if r.Channel != nil { - r.Channel.Close() - } - if r.Conn != nil { - r.Conn.Close() - } -} diff --git a/internal/domain/job-description.go b/internal/domain/job-description.go deleted file mode 100644 index bfe4d99..0000000 --- a/internal/domain/job-description.go +++ /dev/null @@ -1,7 +0,0 @@ -package domain - -type JobDescription struct { - JobID int64 - Description string - Criteria map[string]string -} diff --git a/internal/domain/job-with-description.go b/internal/domain/job-with-description.go deleted file mode 100644 index 6720fb2..0000000 --- a/internal/domain/job-with-description.go +++ /dev/null @@ -1,6 +0,0 @@ -package domain - -type JobWithDescription struct { - Job Job - JobDescription JobDescription -} diff --git a/internal/domain/job.go b/internal/domain/job.go deleted file mode 100644 index 15d495e..0000000 --- a/internal/domain/job.go +++ /dev/null @@ -1,53 +0,0 @@ -package domain - -import "time" - -// JobProvider represents different job board providers -type JobProvider int - -type JobStatus int - -const ( - LinkedIn JobProvider = iota - Indeed - Glassdoor - Bayt - TokyoDev - JapanDev -) - -const ( - JobStatusCreated JobStatus = iota - JobStatusAnalyzed - JobStatusError -) - -type Job struct { - ID int64 - Title string - Company string - CompanyLink string - Location string - JobLink string - Provider JobProvider - JobPostTime *time.Time - Status JobStatus -} - -type JobAnalysisResult struct { - ID int64 - JobID int64 - AnalysisResult string - MatchScore *int - KeySkills []string - MissingSkills []string - Recommendations *string - AnalyzedAt time.Time -} - -type SearchQuery struct { - Keywords string `json:"keywords"` - Location string `json:"location"` - NumPages int `json:"numPages"` - FWT string `json:"f_WT"` // Work type filter (1=onsite, 2=remote, 3=hybrid) -} diff --git a/internal/ports/job-commands.go b/internal/ports/job-commands.go deleted file mode 100644 index 7c9b7e9..0000000 --- a/internal/ports/job-commands.go +++ /dev/null @@ -1,21 +0,0 @@ -package ports - -import "github.com/jobs-scraper/internal/domain" - -type JobCommands struct { - CreateJob JobCommandHandler -} - -// CreateJobCommand represents the command to create a job -type CreateJobCommand struct { - Location string - Keywords string - FWT string - Provider domain.JobProvider - NumPages int -} - -// JobCommandHandler defines the interface for handling job commands -type JobCommandHandler interface { - Handle(cmd CreateJobCommand) error -} diff --git a/internal/repo/job-analysis-result.go b/internal/repo/job-analysis-result.go deleted file mode 100644 index 8f46e37..0000000 --- a/internal/repo/job-analysis-result.go +++ /dev/null @@ -1,152 +0,0 @@ -package repo - -import ( - "database/sql" - "fmt" - - "github.com/jobs-scraper/internal/domain" - "github.com/lib/pq" -) - -type JobAnalysisResultRepository struct { - db *sql.DB -} - -func NewJobAnalysisResultRepository(db *sql.DB) *JobAnalysisResultRepository { - return &JobAnalysisResultRepository{db: db} -} - -func (r *JobAnalysisResultRepository) SaveAnalysisResult(result *domain.JobAnalysisResult) error { - query := ` - INSERT INTO job_analysis_results (job_id, analysis_result, match_score, key_skills, missing_skills, recommendations) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id, analyzed_at - ` - - err := r.db.QueryRow( - query, - result.JobID, - result.AnalysisResult, - result.MatchScore, - pq.Array(result.KeySkills), - pq.Array(result.MissingSkills), - result.Recommendations, - ).Scan(&result.ID, &result.AnalyzedAt) - - if err != nil { - return fmt.Errorf("error saving job analysis result: %v", err) - } - - return nil -} - -func (r *JobAnalysisResultRepository) GetAnalysisResultByJobID(jobID int64) (*domain.JobAnalysisResult, error) { - query := ` - SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at - FROM job_analysis_results - WHERE job_id = $1 - ORDER BY analyzed_at DESC - LIMIT 1 - ` - - var result domain.JobAnalysisResult - err := r.db.QueryRow(query, jobID).Scan( - &result.ID, - &result.JobID, - &result.AnalysisResult, - &result.MatchScore, - pq.Array(&result.KeySkills), - pq.Array(&result.MissingSkills), - &result.Recommendations, - &result.AnalyzedAt, - ) - - if err == sql.ErrNoRows { - return nil, fmt.Errorf("no analysis result found for job ID %d", jobID) - } - - if err != nil { - return nil, fmt.Errorf("error querying job analysis result: %v", err) - } - - return &result, nil -} - -func (r *JobAnalysisResultRepository) GetAllAnalysisResults() ([]domain.JobAnalysisResult, error) { - query := ` - SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at - FROM job_analysis_results - ORDER BY analyzed_at DESC - ` - - rows, err := r.db.Query(query) - if err != nil { - return nil, fmt.Errorf("error querying job analysis results: %v", err) - } - defer rows.Close() - - var results []domain.JobAnalysisResult - for rows.Next() { - var result domain.JobAnalysisResult - err := rows.Scan( - &result.ID, - &result.JobID, - &result.AnalysisResult, - &result.MatchScore, - pq.Array(&result.KeySkills), - pq.Array(&result.MissingSkills), - &result.Recommendations, - &result.AnalyzedAt, - ) - if err != nil { - return nil, fmt.Errorf("error scanning job analysis result: %v", err) - } - results = append(results, result) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err) - } - - return results, nil -} - -func (r *JobAnalysisResultRepository) GetAnalysisResultsByMatchScore(minScore int) ([]domain.JobAnalysisResult, error) { - query := ` - SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at - FROM job_analysis_results - WHERE match_score >= $1 - ORDER BY match_score DESC, analyzed_at DESC - ` - - rows, err := r.db.Query(query, minScore) - if err != nil { - return nil, fmt.Errorf("error querying job analysis results by match score: %v", err) - } - defer rows.Close() - - var results []domain.JobAnalysisResult - for rows.Next() { - var result domain.JobAnalysisResult - err := rows.Scan( - &result.ID, - &result.JobID, - &result.AnalysisResult, - &result.MatchScore, - pq.Array(&result.KeySkills), - pq.Array(&result.MissingSkills), - &result.Recommendations, - &result.AnalyzedAt, - ) - if err != nil { - return nil, fmt.Errorf("error scanning job analysis result: %v", err) - } - results = append(results, result) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err) - } - - return results, nil -} diff --git a/internal/repo/job-description.go b/internal/repo/job-description.go deleted file mode 100644 index 24c0b30..0000000 --- a/internal/repo/job-description.go +++ /dev/null @@ -1,87 +0,0 @@ -package repo - -import ( - "database/sql" - "encoding/json" - "fmt" - "strings" - - "github.com/jobs-scraper/internal/domain" -) - -type JobDescriptionRepository struct { - db *sql.DB -} - -type JobDescriptionData struct { - JobID int - Description string - Criteria map[string]string -} - -func NewJobDescriptionRepository(db *sql.DB) *JobDescriptionRepository { - return &JobDescriptionRepository{db: db} -} - -func (r *JobDescriptionRepository) SaveJobDescriptions(jobDescriptions []domain.JobDescription) error { - if len(jobDescriptions) == 0 { - return nil - } - - // Build the VALUES clause dynamically - valueStrings := make([]string, 0, len(jobDescriptions)) - valueArgs := make([]interface{}, 0, len(jobDescriptions)*3) - - for i, jd := range jobDescriptions { - // Convert criteria map to JSONB - criteriaByte, err := json.Marshal(jd.Criteria) - if err != nil { - return fmt.Errorf("error marshaling job criteria for job %d: %v", jd.JobID, err) - } - - valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i*3+1, i*3+2, i*3+3)) - valueArgs = append(valueArgs, jd.JobID, jd.Description, criteriaByte) - } - - sqlStatement := fmt.Sprintf(` - INSERT INTO job_descriptions (job_id, description, job_criteria) - VALUES %s - ON CONFLICT (job_id) DO UPDATE SET - description = EXCLUDED.description, - job_criteria = EXCLUDED.job_criteria, - updated_at = CURRENT_TIMESTAMP - `, strings.Join(valueStrings, ",")) - - _, err := r.db.Exec(sqlStatement, valueArgs...) - if err != nil { - return fmt.Errorf("error saving job descriptions: %v", err) - } - - return nil -} - -func (r *JobDescriptionRepository) GetJobDescriptionByJobID(jobID int64) (string, map[string]string, error) { - var ( - description string - criteriaByte []byte - criteria map[string]string - ) - - sqlStatement := `SELECT description, job_criteria FROM job_descriptions WHERE job_id = $1` - err := r.db.QueryRow(sqlStatement, jobID).Scan(&description, &criteriaByte) - if err != nil { - if err == sql.ErrNoRows { - return "", nil, nil // No description found - } - return "", nil, fmt.Errorf("error fetching job description: %v", err) - } - - // Unmarshal the JSON criteria - if criteriaByte != nil { - if err := json.Unmarshal(criteriaByte, &criteria); err != nil { - return "", nil, fmt.Errorf("error unmarshaling job criteria: %v", err) - } - } - - return description, criteria, nil -} diff --git a/internal/repo/job.go b/internal/repo/job.go deleted file mode 100644 index 2b9f944..0000000 --- a/internal/repo/job.go +++ /dev/null @@ -1,179 +0,0 @@ -package repo - -import ( - "bytes" - "database/sql" - "fmt" - "strconv" - - "github.com/jobs-scraper/internal/domain" -) - -type JobRepository struct { - db *sql.DB -} - -func NewJobRepository(db *sql.DB) *JobRepository { - return &JobRepository{db: db} -} - -func (r *JobRepository) SaveJobs(jobs []domain.Job) error { - if len(jobs) == 0 { - return nil - } - - // Deduplicate jobs by ID to avoid duplicate errors - jobMap := make(map[int64]domain.Job) - for _, job := range jobs { - jobMap[job.ID] = job - } - - // Convert to slice of pointers for the bulk function - uniqueJobs := make([]*domain.Job, 0, len(jobMap)) - for _, job := range jobMap { - jobCopy := job // Important: create a copy to avoid pointer issues - uniqueJobs = append(uniqueJobs, &jobCopy) - } - - sqlTemplate := ` - INSERT INTO jobs (id, title, company, company_link, location, job_link, job_timestamp) - VALUES %s - ON CONFLICT (id) DO UPDATE SET - title = EXCLUDED.title, - company = EXCLUDED.company, - company_link = EXCLUDED.company_link, - location = EXCLUDED.location, - job_link = EXCLUDED.job_link, - job_timestamp = EXCLUDED.job_timestamp - ` - - sqlStatement, vals := prepareQueryCreateBulk(sqlTemplate, uniqueJobs) - - _, err := r.db.Exec(sqlStatement, vals...) - if err != nil { - return fmt.Errorf("error inserting jobs: %v", err) - } - - return nil -} - -func (r *JobRepository) GetAllJobs() ([]domain.Job, error) { - rows, err := r.db.Query("SELECT * FROM jobs") - if err != nil { - return nil, fmt.Errorf("error querying jobs: %v", err) - } - defer rows.Close() - - var jobs []domain.Job - for rows.Next() { - var job domain.Job - if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil { - return nil, fmt.Errorf("error scanning job row: %v", err) - } - jobs = append(jobs, job) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating over job rows: %v", err) - } - - return jobs, nil -} -func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { - var job domain.Job - - sqlStatement := ` - SELECT id, title, company, company_link, location, job_link, job_timestamp - FROM jobs - WHERE id = $1 - ` - - err := r.db.QueryRow(sqlStatement, id).Scan( - &job.ID, - &job.Title, - &job.Company, - &job.CompanyLink, - &job.Location, - &job.JobLink, - &job.JobPostTime, - ) - - if err == sql.ErrNoRows { - return nil, fmt.Errorf("job with ID %d not found", id) - } - - if err != nil { - return nil, fmt.Errorf("error querying job: %v", err) - } - - return &job, nil -} - -func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) { - bf := bytes.Buffer{} - values := make([]interface{}, 0, len(models)*7) - - for i, v := range models { - values = append(values, v.ID, v.Title, v.Company, - v.CompanyLink, v.Location, v.JobLink, v.JobPostTime, - ) - - numFields := 7 // the number of fields you are inserting - n := i * numFields - - bf.WriteString("(") - for j := 0; j < numFields; j++ { - bf.WriteString("$") - bf.WriteString(strconv.Itoa(n + j + 1)) - if j < numFields-1 { - bf.WriteString(", ") - } - } - bf.WriteString(")") - if i < len(models)-1 { - bf.WriteString(", ") - } - } - - return fmt.Sprintf(s, bf.String()), values -} - -func (r *JobRepository) GetJobsByStatus(status domain.JobStatus) ([]domain.Job, error) { - query := ` - SELECT id, title, company, company_link, location, job_link, job_timestamp - FROM jobs - WHERE status = $1 - ` - - rows, err := r.db.Query(query, int(status)) - if err != nil { - return nil, err - } - defer rows.Close() - - var jobs []domain.Job - for rows.Next() { - var job domain.Job - err := rows.Scan( - &job.ID, - &job.Title, - &job.Company, - &job.CompanyLink, - &job.Location, - &job.JobLink, - &job.JobPostTime, - ) - if err != nil { - return nil, err - } - jobs = append(jobs, job) - } - - return jobs, rows.Err() -} - -func (r *JobRepository) UpdateJobStatus(jobID int64, status domain.JobStatus) error { - query := `UPDATE jobs SET status = $1 WHERE id = $2` - _, err := r.db.Exec(query, int(status), jobID) - return err -} diff --git a/internal/utils/http.go b/internal/utils/http.go deleted file mode 100644 index 7ba79da..0000000 --- a/internal/utils/http.go +++ /dev/null @@ -1,87 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - "log/slog" - "net/http" -) - -// APIError represents an API error with status code and message -type APIError struct { - StatusCode int `json:"statusCode"` - Msg any `json:"msg"` -} - -// Error implements the error interface for APIError -func (e APIError) Error() string { - return fmt.Sprintf("api error: %d", e.StatusCode) -} - -// NewAPIError creates a new APIError with the given status code and error message -func NewAPIError(statusCode int, err error) APIError { - return APIError{ - StatusCode: statusCode, - Msg: err.Error(), - } -} - -// InvalidRequestData creates an APIError for validation errors with a map of field errors -func InvalidRequestData(msg string) APIError { - return APIError{ - StatusCode: http.StatusUnprocessableEntity, - Msg: msg, - } -} - -// InvalidJSON creates an APIError for invalid JSON format -func InvalidJSON() APIError { - return NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid JSON request data")) -} - -// APIResponse represents a standardized API response wrapper -type APIResponse struct { - Success bool `json:"success"` - Data any `json:"data,omitempty"` - Error any `json:"error,omitempty"` -} - -// WriteJSON writes the response as JSON with given status code, wrapped in a data object -func WriteJSON(w http.ResponseWriter, status int, data any) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - - response := APIResponse{ - Success: status >= 200 && status < 300, - } - - if response.Success { - response.Data = data - } else { - response.Error = data - } - - json.NewEncoder(w).Encode(response) -} - -// APIFunc is the signature for API handler functions -type APIFunc func(w http.ResponseWriter, r *http.Request) error - -// Make wraps an APIFunc and handles errors consistently -func Make(h APIFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if err := h(w, r); err != nil { - if apiErr, ok := err.(APIError); ok { - WriteJSON(w, apiErr.StatusCode, apiErr) - } else { - errResp := map[string]any{ - "statusCode": http.StatusInternalServerError, - "msg": "internal server error", - } - WriteJSON(w, http.StatusInternalServerError, errResp) - // You could add logging here - slog.Error("HTTP API error", "err", err.Error(), "path", r.URL.Path) - } - } - } -} diff --git a/internal/utils/retryable-http-request.go b/internal/utils/retryable-http-request.go deleted file mode 100644 index aa0bde1..0000000 --- a/internal/utils/retryable-http-request.go +++ /dev/null @@ -1,425 +0,0 @@ -package utils - -import ( - "context" - // "crypto/tls" - "fmt" - "io" - "net/http" - // "net/url" - "time" -) - -type RetryConfig struct { - MaxRetries int - BaseDelay time.Duration - MaxDelay time.Duration -} - -type RetryableHTTPRequestImpl struct { - client *http.Client - config RetryConfig -} - -func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl { - return NewRetryableHTTPRequestWithProxy(config, "") -} - -func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl { - // var transport *http.Transport - - // if proxyURL != "" { - // proxyUrl, err := url.Parse(proxyURL) - // if err != nil { - // panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err)) - // } - - // transport = &http.Transport{ - // Proxy: http.ProxyURL(proxyUrl), - // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - // } - // } - - client := &http.Client{ - Timeout: 30 * time.Second, - // Transport: transport, - } - - return &RetryableHTTPRequestImpl{ - client: client, - config: config, - } -} - -func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) { - - var lastErr error - - for attempt := 0; attempt <= s.config.MaxRetries; attempt++ { - req, err := http.NewRequestWithContext(ctx, method, url, body) - - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") - for _, header := range headers { - req.Header.Set(header.Get("Key"), header.Get("Value")) - } - - resp, err := s.client.Do(req) - if err != nil { - lastErr = err - fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err) - } else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices { - // Success - return resp, nil - } else if resp.StatusCode == http.StatusTooManyRequests { - // 429 Too Many Requests - retry with backoff - resp.Body.Close() - lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status) - fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode) - } else { - // All other errors (4xx, 5xx) - don't retry - resp.Body.Close() - return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status) - } - - // max attempts for now = 3 - if attempt < s.config.MaxRetries { - // Increment delay by 2 seconds for each attempt - delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ... - - // Cap the delay to MaxDelay if set - if s.config.MaxDelay > 0 && delay > s.config.MaxDelay { - delay = s.config.MaxDelay - } - - fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(delay): - // Continue to next attempt - } - } - } - - return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr) -} - -// 5.10.245.81:80 -// 172.67.70.206:80 -// 190.93.245.33:80 -// 185.221.160.176:80 -// 141.101.123.156:80 -// 103.21.244.129:80 -// 103.169.142.59:80 -// 172.67.81.185:80 -// 37.18.73.60:5566 -// 209.38.83.56:1088 -// 89.169.36.109:1080 -// 103.21.244.248:80 -// 172.67.70.92:80 -// 172.67.88.34:80 -// 141.101.120.38:80 -// 172.67.75.255:80 -// 172.67.75.171:80 -// 45.131.4.157:80 -// 141.101.121.208:80 -// 141.101.120.70:80 -// 103.21.244.70:80 -// 103.21.244.140:80 -// 103.21.244.138:80 -// 154.16.146.43:80 -// 128.199.202.122:3128 -// 160.153.0.28:80 -// 185.170.166.31:80 -// 172.67.70.95:80 -// 141.101.123.91:80 -// 141.101.121.81:80 -// 141.101.121.71:80 -// 141.101.120.224:80 -// 192.111.137.35:4145 -// 45.12.31.20:80 -// 103.21.244.150:80 -// 103.21.244.214:80 -// 103.21.244.29:80 -// 195.85.23.88:80 -// 23.227.39.134:80 -// 66.42.224.229:41679 -// 172.67.73.123:80 -// 79.110.200.27:8000 -// 5.182.34.33:80 -// 103.21.244.174:80 -// 172.67.75.203:80 -// 141.101.122.150:80 -// 141.101.120.35:80 -// 175.47.237.95:6128 -// 103.21.244.185:80 -// 141.101.122.86:80 -// 141.101.121.89:80 -// 103.21.244.105:80 -// 103.21.244.234:80 -// 172.67.70.20:80 -// 163.223.172.27:1080 -// 172.67.84.128:80 -// 103.21.244.57:80 -// 31.43.179.191:80 -// 173.245.59.61:80 -// 141.101.120.48:80 -// 185.162.230.19:80 -// 172.67.172.150:80 -// 159.112.235.63:80 -// 172.64.40.184:80 -// 103.21.244.55:80 -// 103.21.244.49:80 -// 172.67.70.228:80 -// 103.21.244.8:80 -// 66.235.200.77:80 -// 45.85.119.147:80 -// 172.67.177.231:80 -// 173.245.49.28:80 -// 110.232.92.49:8080 -// 212.183.88.212:80 -// 108.162.193.73:80 -// 185.238.228.29:80 -// 172.67.127.8:80 -// 164.38.155.86:80 -// 160.153.0.18:80 -// 141.101.122.119:80 -// 103.21.244.186:80 -// 172.67.172.162:80 -// 8.218.39.40:10800 -// 47.237.132.101:60031 -// 222.59.173.105:44193 -// 143.110.190.60:1080 -// 152.53.194.55:32059 -// 115.187.50.40:5678 -// 171.254.219.180:1080 -// 103.165.157.155:8080 -// 45.233.169.57:999 -// 222.59.173.105:45035 -// 202.40.179.18:4145 -// 170.79.181.188:60606 -// 37.186.66.36:3629 -// 104.200.152.30:4145 -// 106.13.58.110:8888 -// 202.40.181.220:31247 -// 43.135.36.240:80 -// 103.82.27.107:10001 -// 192.252.209.158:4145 -// 142.54.236.97:4145 -// 142.54.239.1:4145 -// 184.170.245.148:4145 -// 68.71.247.130:4145 -// 72.195.34.58:4145 -// 172.65.90.2:80 -// 172.65.90.0:80 -// 36.138.53.26:10019 -// 183.215.23.242:9091 -// 45.131.7.34:80 -// 141.101.120.230:80 -// 172.64.78.193:80 -// 141.101.122.9:80 -// 103.21.244.80:80 -// 103.21.244.147:80 -// 103.21.244.106:80 -// 103.21.244.16:80 -// 199.34.229.210:80 -// 141.101.90.98:80 -// 172.67.70.127:80 -// 103.21.244.221:80 -// 172.67.171.203:80 -// 172.67.181.188:80 -// 72.195.101.99:4145 -// 141.101.120.106:80 -// 103.21.244.180:80 -// 103.21.244.102:80 -// 172.67.70.233:80 -// 108.162.193.107:80 -// 45.131.208.112:80 -// 154.194.12.195:80 -// 69.84.182.23:80 -// 141.101.120.128:80 -// 141.101.120.149:80 -// 141.101.120.232:80 -// 141.101.120.6:80 -// 172.67.98.18:80 -// 141.101.113.20:80 -// 208.65.90.21:4145 -// 198.177.254.131:4145 -// 142.54.237.38:4145 -// 192.252.220.89:4145 -// 199.187.210.54:4145 -// 199.102.105.242:4145 -// 199.102.107.145:4145 -// 198.8.84.3:4145 -// 98.182.171.161:4145 -// 192.111.129.150:4145 -// 184.170.248.5:4145 -// 98.188.47.150:4145 -// 125.228.94.232:4145 -// 125.228.94.153:4145 -// 185.162.229.219:80 -// 103.21.244.22:80 -// 172.64.89.97:80 -// 103.21.244.54:80 -// 103.21.244.37:80 -// 141.101.123.225:80 -// 103.21.244.168:80 -// 103.21.244.149:80 -// 103.21.244.100:80 -// 103.21.244.24:80 -// 23.227.39.209:80 -// 172.67.83.15:80 -// 45.131.4.241:80 -// 141.101.120.201:80 -// 172.67.191.237:80 -// 172.67.127.188:80 -// 103.21.244.92:80 -// 103.160.204.22:80 -// 220.197.44.36:3128 -// 39.185.41.193:5911 -// 23.227.39.121:80 -// 139.162.78.109:80 -// 188.114.99.144:80 -// 45.131.5.37:80 -// 45.131.4.250:80 -// 23.227.39.65:80 -// 141.101.121.191:80 -// 172.67.188.16:80 -// 172.67.254.148:80 -// 221.1.104.177:7302 -// 222.59.173.105:44008 -// 103.21.244.83:80 -// 172.64.149.1:80 -// 45.12.30.22:80 -// 222.59.173.105:44027 -// 172.64.149.26:80 -// 103.21.244.46:80 -// 45.12.31.242:80 -// 23.227.38.195:80 -// 172.67.177.162:80 -// 103.160.204.200:80 -// 172.67.70.129:80 -// 141.101.123.245:80 -// 185.162.230.117:80 -// 185.162.230.183:80 -// 69.61.200.104:36181 -// 208.65.90.3:4145 -// 72.223.188.92:4145 -// 192.252.214.17:4145 -// 68.71.242.118:4145 -// 192.252.210.233:4145 -// 107.181.161.81:4145 -// 107.181.168.145:4145 -// 206.220.175.2:4145 -// 72.37.217.3:4145 -// 192.252.208.70:14282 -// 70.166.167.55:57745 -// 192.111.137.37:18762 -// 98.188.47.132:4145 -// 62.99.138.162:80 -// 172.64.90.186:80 -// 172.67.74.57:80 -// 45.131.6.67:80 -// 141.101.122.37:80 -// 103.21.244.134:80 -// 172.64.155.71:80 -// 5.182.34.139:80 -// 45.131.6.31:80 -// 141.101.121.21:80 -// 141.101.120.190:80 -// 103.21.244.161:80 -// 103.21.244.156:80 -// 103.21.244.151:80 -// 172.64.89.0:80 -// 58.216.109.17:800 -// 58.241.88.18:800 -// 36.147.78.166:80 -// 170.244.26.36:8888 -// 170.244.25.52:8888 -// 170.244.26.206:8888 -// 36.138.53.26:10017 -// 201.148.32.162:80 -// 203.19.38.114:1080 -// 31.43.179.204:80 -// 45.12.31.50:80 -// 170.244.27.142:8888 -// 170.244.26.195:8888 -// 170.244.27.58:8888 -// 170.244.27.61:8888 -// 170.244.27.150:8888 -// 47.243.94.125:1080 -// 40.177.65.8:80 -// 47.57.13.107:80 -// 194.158.203.14:80 -// 194.219.134.234:80 -// 103.21.244.51:80 -// 103.21.244.160:80 -// 103.21.244.144:80 -// 213.33.126.130:80 -// 103.21.244.69:80 -// 103.21.244.192:80 -// 103.21.244.133:80 -// 141.193.213.189:80 -// 68.71.254.6:4145 -// 185.221.160.21:80 -// 188.114.99.97:80 -// 63.141.128.94:80 -// 141.193.213.213:80 -// 185.162.228.234:80 -// 213.143.113.82:80 -// 154.194.12.207:80 -// 189.203.181.34:1080 -// 202.144.134.150:5678 -// 190.242.157.215:8080 -// 72.49.49.11:31034 -// 142.54.237.34:4145 -// 68.71.249.153:48606 -// 192.252.209.155:14455 -// 192.252.216.86:4145 -// 142.54.229.249:4145 -// 209.97.150.167:8080 -// 192.252.220.92:17328 -// 98.178.72.21:10919 -// 144.124.228.87:1080 -// 183.240.46.42:80 -// 32.223.6.94:80 -// 192.252.214.20:15864 -// 198.177.252.24:4145 -// 192.252.208.67:14287 -// 198.8.94.170:4145 -// 68.71.241.33:4145 -// 98.181.137.83:4145 -// 45.12.30.139:80 -// 103.21.244.97:80 -// 65.1.148.157:80 -// 199.58.184.97:4145 -// 141.101.120.148:80 -// 185.238.228.77:80 -// 172.67.202.134:80 -// 172.67.182.49:80 -// 45.131.210.112:80 -// 141.101.120.100:80 -// 103.21.244.59:80 -// 103.21.244.189:80 -// 103.21.244.154:80 -// 103.21.244.23:80 -// 173.245.49.40:80 -// 82.200.235.134:38191 -// 43.224.118.89:2626 -// 119.148.47.226:16464 -// 211.230.49.122:3128 -// 62.171.159.232:8888 -// 115.127.112.34:1080 -// 185.191.236.162:3128 -// 41.223.119.156:3128 -// 103.138.123.242:8082 -// 121.169.46.116:1090 -// 185.145.185.218:8080 -// 45.166.93.113:999 -// 163.53.204.178:9813 -// 181.224.226.154:8080 diff --git a/internal/services/openrouter.go b/shared/openrouter/openrouter.go similarity index 99% rename from internal/services/openrouter.go rename to shared/openrouter/openrouter.go index afa346a..76654fb 100644 --- a/internal/services/openrouter.go +++ b/shared/openrouter/openrouter.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/eduardolat/openroutergo" - "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/shared/domain" ) // JobAnalysisResult represents the structured response from job analysis diff --git a/shared/server/go.mod b/shared/server/go.mod new file mode 100644 index 0000000..1f3a369 --- /dev/null +++ b/shared/server/go.mod @@ -0,0 +1,7 @@ +module github.com/jobs-scraper/shared/server + +go 1.24.0 + +require github.com/gorilla/mux v1.8.1 + +replace github.com/jobs-scraper/shared/server => ./ diff --git a/shared/server/go.sum b/shared/server/go.sum new file mode 100644 index 0000000..7128337 --- /dev/null +++ b/shared/server/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/internal/server/server.go b/shared/server/server.go similarity index 100% rename from internal/server/server.go rename to shared/server/server.go diff --git a/docs/docs.go b/swagger/docs.go similarity index 100% rename from docs/docs.go rename to swagger/docs.go diff --git a/docs/swagger.json b/swagger/swagger.json similarity index 100% rename from docs/swagger.json rename to swagger/swagger.json diff --git a/docs/swagger.yaml b/swagger/swagger.yaml similarity index 100% rename from docs/swagger.yaml rename to swagger/swagger.yaml