package fetcher import( "project.hechon.fr/internal/config" "log" "net/http" "github.com/PuerkitoBio/goquery" "strconv" "strings" "fmt" ) func FetchChapter(cfg *config.Cfg , searchedChapter string) ( error){ url := "https://www.biblegateway.com/passage/?search=" + cfg.Book.Name + "%20" + searchedChapter +"&version=" + cfg.Book.Version res, err := http.Get(url) if err != nil { log.Printf("can’t get %s : %v\n", url, err) return err } defer res.Body.Close() if res.StatusCode != http.StatusOK { log.Printf("error %v\n", res.Status) return err } doc, err := goquery.NewDocumentFromReader(res.Body) if err != nil { log.Printf("error reading the http response: %v\n", err) return err } chapterNum, err := strconv.Atoi(searchedChapter) if err != nil{ log.Printf("searched chapter %v is not a valid number - %v", searchedChapter, err) } chapterText := config.Chapter{ Number: chapterNum, } replacer := strings.NewReplacer( "L'Éternel", "YAHWEH", "l'Éternel", "YAHWEH", "Éternel", "YAHWEH", ) doc.Find(".verse > span").Each(func(i int, s *goquery.Selection){ /**********************************************************************************/ /**** verses are marked by an .verse class. Inside the text is in a span and *****/ /**** begin either by the chapter number (first verse), or by the verse *****/ /**** (the other verses). With that the code is straight forward. *****/ /**********************************************************************************/ rawText := s.Text() verse:= "" if i == 0 { verse, _ = strings.CutPrefix(rawText, searchedChapter ) }else{ verse, _ = strings.CutPrefix(rawText, strconv.Itoa(i+1)) } verse = replacer.Replace(verse) chapterText.Verses = append(chapterText.Verses, strings.TrimSpace(verse)) }) chapterText = StripAlternativeVersification(chapterText) cfg.Book.Chapters = append(cfg.Book.Chapters, chapterText) err= SaveChapter(*cfg, &chapterText) if err != nil{ log.Printf("Error saving chapter %v in yaml format.", chapterNum) return fmt.Errorf("Error saving chapter %v in yaml format.", chapterNum) } return nil } func FetchBook (cfg *config.Cfg) error{ chapterNum, exist := cfg.BookList[cfg.Book.Name] if !exist { log.Printf("%v is not a Bible book or has not been inserted yet.", cfg.Book.Name) return fmt.Errorf("%v is not a Bible book or has not been inserted yet.", cfg.Book.Name) } for i:= 1 ; i <= chapterNum ; i++{ err := FetchChapter(cfg, strconv.Itoa(i)) if err != nil { log.Printf("Warning : chapter %v has not been fetched: %v", i, err) continue } } return nil }