2ab810be72
internal/ and libs/ were both shared-code roots with no rule for which
one a package belonged in, so the split carried no information. Adopt a
rule the language answers by itself:
shared Go -> internal/
shared TypeScript -> libs/ (npm workspace packages)
libs/{ports,repo,server} move to internal/, leaving libs/ holding only
the two TypeScript packages (@jobs-scraper/rabbitmq-ts and
@jobs-scraper/browser-automation), which matches npm workspace
convention. internal/ is also Go's marker for code not importable from
outside the repo, which is accurate here since none of it is published.
Import paths are rewritten mechanically (jobs-scraper/libs/ ->
jobs-scraper/internal/) across 15 lines in 10 files. The 6 moved files
are pure renames with no content change. Doing this after the module
collapse in the previous commit meant no go.mod or replace-directive
edits were needed.
gofmt is applied to services/api/pkg/http/job.go, whose import group the
rewrite left out of order. Three files were already unformatted before
this refactor (internal/openai/interface.go, internal/ports/job-queries.go,
internal/repo/job-analysis-result.go) and are deliberately left alone to
keep this diff limited to the move.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
65 lines
1.9 KiB
Go
65 lines
1.9 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
// "log"
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
"github.com/jobs-scraper/internal/domain"
|
|
"github.com/jobs-scraper/internal/repo"
|
|
)
|
|
|
|
// JobDescriptionResult represents the result of job description scraping
|
|
type JobDescriptionResult struct {
|
|
Job domain.Job
|
|
Description string
|
|
Criteria map[string]string
|
|
Error error
|
|
}
|
|
|
|
// JobPipeline manages the job processing pipeline
|
|
type JobPipeline struct {
|
|
scraperService *Scraper
|
|
numWorkers int
|
|
rateLimit time.Duration
|
|
}
|
|
|
|
// NewJobPipeline creates a new job processing pipeline
|
|
func NewJobPipeline(scraperService *Scraper, numWorkers int, rateLimit time.Duration) *JobPipeline {
|
|
return &JobPipeline{
|
|
scraperService: scraperService,
|
|
numWorkers: numWorkers,
|
|
rateLimit: rateLimit,
|
|
}
|
|
}
|
|
|
|
// ProcessJobsStreaming processes jobs and job descriptions concurrently
|
|
func (p *JobPipeline) ProcessJobsStreaming(ctx context.Context, jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository, searchQuery domain.SearchQuery) error {
|
|
allJobs := make([]domain.Job, 0, 100)
|
|
allJobDescriptions := make([]domain.JobDescription, 0, 100)
|
|
var jbMu sync.Mutex
|
|
jobsChan := GetJobs(ctx, p.scraperService, searchQuery)
|
|
jobWithDescriptionChan := GetJobDescription(ctx, p.scraperService, jobsChan, p.numWorkers)
|
|
|
|
for jobWithDescription := range jobWithDescriptionChan {
|
|
fmt.Printf("Received job description for job : %d\n", jobWithDescription.Job.ID)
|
|
jbMu.Lock()
|
|
allJobs = append(allJobs, jobWithDescription.Job)
|
|
allJobDescriptions = append(allJobDescriptions, jobWithDescription.JobDescription)
|
|
jbMu.Unlock()
|
|
}
|
|
|
|
if err := jobRepo.SaveJobs(allJobs); err != nil {
|
|
return fmt.Errorf("failed to save jobs to database: %w", err)
|
|
}
|
|
|
|
if err := jobDescRepo.SaveJobDescriptions(allJobDescriptions); err != nil {
|
|
return fmt.Errorf("failed to save job descriptions: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|