diff --git a/internal/dto/jobs.go b/internal/dto/jobs.go new file mode 100644 index 0000000..e97a69a --- /dev/null +++ b/internal/dto/jobs.go @@ -0,0 +1,37 @@ +package dto + +import "github.com/jobs-scraper/internal/pkg/domain" + +type JobDTO struct { + ID int64 `json:"id"` + Title string `json:"title"` + Company string `json:"company"` + CompanyLink string `json:"companyLink"` + Location string `json:"location"` + JobLink string `json:"jobLink"` + JobPostTime string `json:"jobPostTime,omitempty"` +} + +func JobFromDomain(j domain.Job) JobDTO { + var postTime string + if j.JobPostTime != nil { + postTime = j.JobPostTime.Format("2006-01-02") + } + return JobDTO{ + ID: j.ID, + Title: j.Title, + Company: j.Company, + CompanyLink: j.CompanyLink, + Location: j.Location, + JobLink: j.JobLink, + JobPostTime: postTime, + } +} + +func JobsFromDomain(jobs []domain.Job) []JobDTO { + dtos := make([]JobDTO, len(jobs)) + for i, j := range jobs { + dtos[i] = JobFromDomain(j) + } + return dtos +} diff --git a/internal/migrations/009_job_indexes.down.sql b/internal/migrations/009_job_indexes.down.sql new file mode 100644 index 0000000..1534ad3 --- /dev/null +++ b/internal/migrations/009_job_indexes.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_jobs_provider_location; \ No newline at end of file diff --git a/internal/migrations/009_job_indexes.up.sql b/internal/migrations/009_job_indexes.up.sql new file mode 100644 index 0000000..3e6ad36 --- /dev/null +++ b/internal/migrations/009_job_indexes.up.sql @@ -0,0 +1 @@ +CREATE INDEX idx_jobs_provider_location ON jobs(provider, location); \ No newline at end of file diff --git a/internal/pkg/domain/job.go b/internal/pkg/domain/job.go index 43cc2a4..67a4537 100644 --- a/internal/pkg/domain/job.go +++ b/internal/pkg/domain/job.go @@ -1,6 +1,8 @@ package domain -import "time" +import ( + "time" +) // JobProvider represents different job board providers type JobProvider int diff --git a/internal/pkg/utils/pagination.go b/internal/pkg/utils/pagination.go new file mode 100644 index 0000000..4494abd --- /dev/null +++ b/internal/pkg/utils/pagination.go @@ -0,0 +1,26 @@ +package utils + +import "math" + +type PageViewModel struct { + Total int `json:"total"` + PageNumber int `json:"pageNumber"` + PageSize int `json:"pageSize"` + TotalPages int `json:"totalPages"` + HasPrevious bool `json:"hasPrevious"` + HasNext bool `json:"hasNext"` +} + +// NewPageViewModel creates a new PageViewModel with calculated fields +func NewPageViewModel(count, pageNumber, pageSize int) *PageViewModel { + totalPages := int(math.Ceil(float64(count) / float64(pageSize))) + + return &PageViewModel{ + Total: count, + PageNumber: pageNumber, + PageSize: pageSize, + TotalPages: totalPages, + HasPrevious: pageNumber > 1, + HasNext: pageNumber < totalPages, + } +} diff --git a/libs/ports/job-queries.go b/libs/ports/job-queries.go new file mode 100644 index 0000000..6a37de0 --- /dev/null +++ b/libs/ports/job-queries.go @@ -0,0 +1,25 @@ +package ports + +import ( + "github.com/jobs-scraper/internal/dto" + "github.com/jobs-scraper/internal/pkg/domain" +) + +type JobQueries struct { + GetJobs JobQueryHandler +} + +// GetJobQuery represents the command to create a job +type GetJobQuery struct { + Location string `schema:"location"` + Keywords string `schema:"keywords"` + FWT string `schema:"fwt"` + Provider domain.JobProvider `schema:"provider"` + PageNumber int `schema:"pageNumber"` + PageSize int `schema:"pageSize"` +} + +// JobQueryHandler defines the interface for handling job queries +type JobQueryHandler interface { + Handle(query GetJobQuery) ([]dto.JobDTO, int, error) +} diff --git a/libs/repo/job.go b/libs/repo/job.go index e75044a..650f4d7 100644 --- a/libs/repo/job.go +++ b/libs/repo/job.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "strconv" + "strings" "github.com/jobs-scraper/internal/pkg/domain" ) @@ -94,27 +95,86 @@ func (r *JobRepository) SaveJobs(jobs []domain.Job) error { return nil } -func (r *JobRepository) GetAllJobs() ([]domain.Job, error) { - rows, err := r.db.Query("SELECT * FROM jobs") +type JobFilter struct { + Location string + Keywords string + Provider domain.JobProvider + PageSize int + PageNumber int +} + +func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) { + query := "SELECT id, title, company, company_link, location, job_link, job_timestamp FROM jobs" + countQuery := "SELECT COUNT(*) FROM jobs" + var conditions []string + var args []interface{} + paramIdx := 1 + + if filter.Location != "" { + conditions = append(conditions, fmt.Sprintf("location ILIKE $%d", paramIdx)) + args = append(args, "%"+filter.Location+"%") + paramIdx++ + } + + if filter.Keywords != "" { + conditions = append(conditions, fmt.Sprintf("(title ILIKE $%d OR company ILIKE $%d)", paramIdx, paramIdx)) + args = append(args, "%"+filter.Keywords+"%") + paramIdx++ + } + + if filter.Provider > 0 { + conditions = append(conditions, fmt.Sprintf("provider = $%d", paramIdx)) + args = append(args, filter.Provider) + paramIdx++ + } + + whereClause := "" + if len(conditions) > 0 { + whereClause = " WHERE " + strings.Join(conditions, " AND ") + query += whereClause + countQuery += whereClause + } + + var totalCount int + countArgs := make([]interface{}, len(args)) + copy(countArgs, args) + if err := r.db.QueryRow(countQuery, countArgs...).Scan(&totalCount); err != nil { + return nil, 0, fmt.Errorf("error counting jobs: %v", err) + } + + if filter.PageSize > 0 { + query += fmt.Sprintf(" LIMIT $%d", paramIdx) + args = append(args, filter.PageSize) + paramIdx++ + + if filter.PageNumber > 0 { + offset := (filter.PageNumber - 1) * filter.PageSize + query += fmt.Sprintf(" OFFSET $%d", paramIdx) + args = append(args, offset) + paramIdx++ + } + } + + rows, err := r.db.Query(query, args...) if err != nil { - return nil, fmt.Errorf("error querying jobs: %v", err) + return nil, 0, fmt.Errorf("error querying jobs: %v", err) } defer rows.Close() var jobs []domain.Job for rows.Next() { var job domain.Job - if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil { - return nil, fmt.Errorf("error scanning job row: %v", err) + if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime); err != nil { + return nil, 0, fmt.Errorf("error scanning job row: %v", err) } jobs = append(jobs, job) } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating over job rows: %v", err) + return nil, 0, fmt.Errorf("error iterating over job rows: %v", err) } - return jobs, nil + return jobs, totalCount, nil } func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { var job domain.Job diff --git a/services/api/cmd/server/main.go b/services/api/cmd/server/main.go index dfe7582..8cbc436 100644 --- a/services/api/cmd/server/main.go +++ b/services/api/cmd/server/main.go @@ -60,6 +60,8 @@ func main() { log.Fatalf("Failed to run migrations: %v", err) } + log.Println("Successfully ran migrations") + router := mux.NewRouter() // Swagger endpoint @@ -73,7 +75,7 @@ func main() { // Create job analysis result repository jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) - jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo) + jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo, app.JobQueries) jobHandler.RegisterRoutes(router) diff --git a/services/api/go.mod b/services/api/go.mod index e1a851f..872ff05 100644 --- a/services/api/go.mod +++ b/services/api/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-openapi/spec v0.20.6 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/gorilla/schema v1.4.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/services/api/go.sum b/services/api/go.sum index 9bf3e82..e8e9f05 100644 --- a/services/api/go.sum +++ b/services/api/go.sum @@ -44,6 +44,8 @@ github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9Knoi github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= +github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -56,6 +58,7 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -113,7 +116,9 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= @@ -122,10 +127,12 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/services/api/internal/app/app.go b/services/api/internal/app/app.go index 4b7e9ed..95c7bbc 100644 --- a/services/api/internal/app/app.go +++ b/services/api/internal/app/app.go @@ -4,11 +4,12 @@ import ( "database/sql" "github.com/gorilla/mux" - "github.com/jobs-scraper/services/api/internal/commands/job" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/server" + jobCommands "github.com/jobs-scraper/services/api/internal/commands/job" + jobQueries "github.com/jobs-scraper/services/api/internal/queries/job" ) type Application struct { @@ -16,6 +17,7 @@ type Application struct { DB *sql.DB Server *server.AppServer JobCommands *ports.JobCommands + JobQueries *ports.JobQueries } func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient) *Application { @@ -28,7 +30,10 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient Router: router, DB: db, JobCommands: &ports.JobCommands{ - CreateJob: job.NewCreateJobHandler(jobRepo, rmq), + CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq), + }, + JobQueries: &ports.JobQueries{ + GetJobs: jobQueries.NewGetJobsHandler(jobRepo), }, } } diff --git a/services/api/internal/queries/job/get-jobs.go b/services/api/internal/queries/job/get-jobs.go new file mode 100644 index 0000000..b08f6c0 --- /dev/null +++ b/services/api/internal/queries/job/get-jobs.go @@ -0,0 +1,34 @@ +package job + +import ( + "github.com/jobs-scraper/internal/dto" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" +) + +type GetJobs struct { + jobRepo *repo.JobRepository +} + +func NewGetJobsHandler(jobRepo *repo.JobRepository) *GetJobs { + return &GetJobs{ + jobRepo: jobRepo, + } +} + +func (cj *GetJobs) Handle(query ports.GetJobQuery) ([]dto.JobDTO, int, error) { + filter := repo.JobFilter{ + Location: query.Location, + Keywords: query.Keywords, + Provider: query.Provider, + PageSize: query.PageSize, + PageNumber: query.PageNumber, + } + + results, totalCount, err := cj.jobRepo.GetAllJobs(filter) + + if err != nil { + return nil, 0, err + } + return dto.JobsFromDomain(results), totalCount, nil +} diff --git a/services/api/pkg/http/job.go b/services/api/pkg/http/job.go index 1dae58b..f15f1ef 100644 --- a/services/api/pkg/http/job.go +++ b/services/api/pkg/http/job.go @@ -7,27 +7,31 @@ import ( "net/http" "strconv" + "github.com/gorilla/mux" + "github.com/gorilla/schema" + "github.com/jobs-scraper/internal/pkg/utils" "github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/repo" - "github.com/jobs-scraper/internal/pkg/utils" - - "github.com/gorilla/mux" ) // JobHandler handles HTTP requests related to jobs type JobHandler struct { jobCommands *ports.JobCommands jobAnalysisResultRepo *repo.JobAnalysisResultRepository + jobQueries *ports.JobQueries } // NewJobHandler creates a new instance of JobHandler -func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler { +func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository, jobQueries *ports.JobQueries) *JobHandler { return &JobHandler{ jobCommands: jobCommands, jobAnalysisResultRepo: jobAnalysisResultRepo, + jobQueries: jobQueries, } } +var decoder = schema.NewDecoder() + // RegisterRoutes registers all routes to the router func (h *JobHandler) RegisterRoutes(router *mux.Router) { jobs := router.PathPrefix("/jobs").Subrouter() @@ -38,6 +42,7 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) { jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET") jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") + jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET") } // CreateJob handles the creation of a new job @@ -68,6 +73,62 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { return nil } +// GetJobs handles the retrieval of all jobs +// @Summary Gets all jobs +// @Description Gets all jobs with the provided specifications +// @Tags jobs +// @Param location query string false "Location filter" +// @Param keywords query string false "Keywords filter (searches title and company)" +// @Param fwt query string false "Full/Part time filter" +// @Param provider query int false "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)" Enums(0,1,2,3,4,5,6,7) +// @Param pageNumber query int false "Page number" +// @Param pageSize query int false "Number of elements" +// @Accept json +// @Produce json +// @Success 201 {object} map[string]string "Successfully retrieved jobs" +// @Failure 400 {object} map[string]string "Invalid request data" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs [get] +func (h *JobHandler) GetJobs(w http.ResponseWriter, r *http.Request) error { + query := ports.GetJobQuery{ + PageNumber: 1, + PageSize: 10, + } + + if pageNumberStr := r.URL.Query().Get("pageNumber"); pageNumberStr != "" { + if page, err := strconv.Atoi(pageNumberStr); err == nil && page > 0 { + query.PageNumber = page + } + } + + if pageSizeStr := r.URL.Query().Get("pageSize"); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil && pageSize > 0 { + query.PageSize = pageSize + } + } + + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + slog.Error("Failed to decode query params", "error", err) + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid query parameters: %w", err)) + } + + result, totalCount, err := h.jobQueries.GetJobs.Handle(query) + + if err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + pagination := utils.NewPageViewModel(totalCount, query.PageNumber, query.PageSize) + + utils.WriteJSON(w, http.StatusOK, map[string]any{ + "jobs": result, + "pagination": pagination, + }) + + return nil +} + // GetAllAnalysisResults handles retrieving all job analysis results // @Summary Get all job analysis results // @Description Retrieves all job analysis results from the database @@ -128,7 +189,7 @@ func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error { minScoreStr := r.URL.Query().Get("min_score") minScore := 70 // default minimum score - + if minScoreStr != "" { var err error minScore, err = strconv.Atoi(minScoreStr) diff --git a/services/api/pkg/swagger/docs.go b/services/api/pkg/swagger/docs.go index 1a64398..1cf7ef5 100644 --- a/services/api/pkg/swagger/docs.go +++ b/services/api/pkg/swagger/docs.go @@ -16,6 +16,96 @@ const docTemplate = `{ "basePath": "{{.BasePath}}", "paths": { "/jobs": { + "get": { + "description": "Gets all jobs with the provided specifications", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Gets all jobs", + "parameters": [ + { + "type": "string", + "description": "Location filter", + "name": "location", + "in": "query" + }, + { + "type": "string", + "description": "Keywords filter (searches title and company)", + "name": "keywords", + "in": "query" + }, + { + "type": "string", + "description": "Full/Part time filter", + "name": "fwt", + "in": "query" + }, + { + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "type": "integer", + "description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)", + "name": "provider", + "in": "query" + }, + { + "type": "integer", + "description": "Page number", + "name": "pageNumber", + "in": "query" + }, + { + "type": "integer", + "description": "Number of elements", + "name": "pageSize", + "in": "query" + } + ], + "responses": { + "201": { + "description": "Successfully retrieved jobs", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + }, "post": { "description": "Creates a new job with the provided specifications", "consumes": [ @@ -254,7 +344,9 @@ const docTemplate = `{ 2, 3, 4, - 5 + 5, + 6, + 7 ], "x-enum-varnames": [ "LinkedIn", @@ -262,7 +354,9 @@ const docTemplate = `{ "Glassdoor", "Bayt", "TokyoDev", - "JapanDev" + "JapanDev", + "Google", + "HiringCafe" ] }, "ports.CreateJobCommand": { diff --git a/services/api/pkg/swagger/swagger.json b/services/api/pkg/swagger/swagger.json index c6c28fd..8ec2ac5 100644 --- a/services/api/pkg/swagger/swagger.json +++ b/services/api/pkg/swagger/swagger.json @@ -9,6 +9,96 @@ "basePath": "/", "paths": { "/jobs": { + "get": { + "description": "Gets all jobs with the provided specifications", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Gets all jobs", + "parameters": [ + { + "type": "string", + "description": "Location filter", + "name": "location", + "in": "query" + }, + { + "type": "string", + "description": "Keywords filter (searches title and company)", + "name": "keywords", + "in": "query" + }, + { + "type": "string", + "description": "Full/Part time filter", + "name": "fwt", + "in": "query" + }, + { + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "type": "integer", + "description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)", + "name": "provider", + "in": "query" + }, + { + "type": "integer", + "description": "Page number", + "name": "pageNumber", + "in": "query" + }, + { + "type": "integer", + "description": "Number of elements", + "name": "pageSize", + "in": "query" + } + ], + "responses": { + "201": { + "description": "Successfully retrieved jobs", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + }, "post": { "description": "Creates a new job with the provided specifications", "consumes": [ @@ -247,7 +337,9 @@ 2, 3, 4, - 5 + 5, + 6, + 7 ], "x-enum-varnames": [ "LinkedIn", @@ -255,7 +347,9 @@ "Glassdoor", "Bayt", "TokyoDev", - "JapanDev" + "JapanDev", + "Google", + "HiringCafe" ] }, "ports.CreateJobCommand": { diff --git a/services/api/pkg/swagger/swagger.yaml b/services/api/pkg/swagger/swagger.yaml index 7040064..103a931 100644 --- a/services/api/pkg/swagger/swagger.yaml +++ b/services/api/pkg/swagger/swagger.yaml @@ -33,6 +33,8 @@ definitions: - 3 - 4 - 5 + - 6 + - 7 type: integer x-enum-varnames: - LinkedIn @@ -41,6 +43,8 @@ definitions: - Bayt - TokyoDev - JapanDev + - Google + - HiringCafe ports.CreateJobCommand: properties: fwt: @@ -61,6 +65,69 @@ info: version: "1.0" paths: /jobs: + get: + consumes: + - application/json + description: Gets all jobs with the provided specifications + parameters: + - description: Location filter + in: query + name: location + type: string + - description: Keywords filter (searches title and company) + in: query + name: keywords + type: string + - description: Full/Part time filter + in: query + name: fwt + type: string + - description: Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, + JapanDev=5, Google=6, HiringCafe=7) + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + in: query + name: provider + type: integer + - description: Page number + in: query + name: pageNumber + type: integer + - description: Number of elements + in: query + name: pageSize + type: integer + produces: + - application/json + responses: + "201": + description: Successfully retrieved jobs + schema: + additionalProperties: + type: string + type: object + "400": + description: Invalid request data + schema: + additionalProperties: + type: string + type: object + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: Gets all jobs + tags: + - jobs post: consumes: - application/json diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go index 1ded428..498d820 100644 --- a/services/scraper-google/main.go +++ b/services/scraper-google/main.go @@ -48,8 +48,8 @@ func main() { sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) - pages := []int{0, 10, 20, 30, 40} - // pages := []int{100} + pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + // pages := []int{50, 60, 70, 80, 90, 100} log.Println("Starting scraper with scheduler...") @@ -58,7 +58,6 @@ func main() { log.Printf("Running initial scrape for page offset: %d, inital run", page) Work(ctx, browser)(page, googleLinkStream) }() - } for _, page := range pages {