upd
This commit is contained in:
@@ -1,11 +1,16 @@
|
|||||||
package capture
|
package capture
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"mime"
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
@@ -13,6 +18,8 @@ import (
|
|||||||
"git.gulenok.ru/greenhaze/gaslight/internal/domain"
|
"git.gulenok.ru/greenhaze/gaslight/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const byURLDir = "by-url"
|
||||||
|
|
||||||
type ImageSaver struct {
|
type ImageSaver struct {
|
||||||
outputDir string
|
outputDir string
|
||||||
counter uint64
|
counter uint64
|
||||||
@@ -23,12 +30,67 @@ func NewImageSaver(outputDir string) *ImageSaver {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *ImageSaver) Save(resp *http.Response, body []byte) error {
|
func (s *ImageSaver) Save(resp *http.Response, body []byte) error {
|
||||||
|
if resp == nil || resp.Request == nil {
|
||||||
|
return fmt.Errorf("nil response or request")
|
||||||
|
}
|
||||||
|
|
||||||
n := atomic.AddUint64(&s.counter, 1)
|
n := atomic.AddUint64(&s.counter, 1)
|
||||||
ts := time.Now().UTC().Format("20060102T150405.000000000Z")
|
ts := time.Now().UTC().Format("20060102T150405.000000000Z")
|
||||||
host := sanitizeName(domain.HostWithoutPort(resp.Request.Host))
|
host := sanitizeName(domain.HostWithoutPort(resp.Request.Host))
|
||||||
ext := imageExtension(resp.Header.Get("Content-Type"), resp.Request.URL.Path)
|
ext := imageExtension(resp.Header.Get("Content-Type"), resp.Request.URL.Path)
|
||||||
name := fmt.Sprintf("%s_%06d_%s%s", ts, n, host, ext)
|
name := fmt.Sprintf("%s_%06d_%s%s", ts, n, host, ext)
|
||||||
return os.WriteFile(filepath.Join(s.outputDir, name), body, 0o644)
|
if err := os.WriteFile(filepath.Join(s.outputDir, name), body, 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
relPath, err := relativeImagePath(resp.Request.Host, resp.Request.URL.Path, resp.Request.URL.RawQuery, resp.Header.Get("Content-Type"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
localPath := filepath.Join(s.outputDir, byURLDir, relPath)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(localPath, body, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ImageSaver) LocalPathByRelativeURL(host, relativeURL string) (string, error) {
|
||||||
|
if strings.TrimSpace(relativeURL) == "" {
|
||||||
|
return "", fmt.Errorf("relative url is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(relativeURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedHost := host
|
||||||
|
if parsed.Host != "" {
|
||||||
|
resolvedHost = parsed.Host
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedPath := parsed.Path
|
||||||
|
if resolvedPath == "" {
|
||||||
|
resolvedPath = "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
localPath, err := resolveByRelativeURL(s.outputDir, resolvedHost, resolvedPath, parsed.RawQuery)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return localPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ImageSaver) ReadByRelativeURL(host, relativeURL string) ([]byte, string, error) {
|
||||||
|
localPath, err := s.LocalPathByRelativeURL(host, relativeURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return data, localPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func IsImageResponse(contentType, path string) bool {
|
func IsImageResponse(contentType, path string) bool {
|
||||||
@@ -43,8 +105,117 @@ func IsImageResponse(contentType, path string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func relativeImagePath(host, urlPath, rawQuery, contentType string) (string, error) {
|
||||||
|
hostDir := sanitizeName(domain.HostWithoutPort(host))
|
||||||
|
dirSegments, baseName, baseExt := splitAndSanitizeURLPath(urlPath)
|
||||||
|
ext := baseExt
|
||||||
|
if ext == "" {
|
||||||
|
ext = imageExtension(contentType, urlPath)
|
||||||
|
}
|
||||||
|
suffix := querySuffix(rawQuery)
|
||||||
|
fileName := baseName + suffix + ext
|
||||||
|
parts := append([]string{hostDir}, dirSegments...)
|
||||||
|
parts = append(parts, fileName)
|
||||||
|
return filepath.Join(parts...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveByRelativeURL(outputDir, host, urlPath, rawQuery string) (string, error) {
|
||||||
|
hostDir := sanitizeName(domain.HostWithoutPort(host))
|
||||||
|
dirSegments, baseName, baseExt := splitAndSanitizeURLPath(urlPath)
|
||||||
|
suffix := querySuffix(rawQuery)
|
||||||
|
baseDir := filepath.Join(append([]string{outputDir, byURLDir, hostDir}, dirSegments...)...)
|
||||||
|
prefix := baseName + suffix
|
||||||
|
|
||||||
|
if baseExt != "" {
|
||||||
|
candidate := filepath.Join(baseDir, prefix+strings.ToLower(baseExt))
|
||||||
|
if fileExists(candidate) {
|
||||||
|
return candidate, nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
candidate := filepath.Join(baseDir, prefix+".img")
|
||||||
|
if fileExists(candidate) {
|
||||||
|
return candidate, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
matches, err := findMatchingFiles(baseDir, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(matches) == 0 {
|
||||||
|
return "", os.ErrNotExist
|
||||||
|
}
|
||||||
|
return matches[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findMatchingFiles(dir, prefix string) ([]string, error) {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var matches []string
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := entry.Name()
|
||||||
|
if strings.HasPrefix(name, prefix+".") {
|
||||||
|
matches = append(matches, filepath.Join(dir, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(matches)
|
||||||
|
return matches, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileExists(path string) bool {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
return err == nil && !info.IsDir()
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitAndSanitizeURLPath(urlPath string) ([]string, string, string) {
|
||||||
|
cleanPath := path.Clean("/" + strings.TrimSpace(urlPath))
|
||||||
|
if cleanPath == "/" {
|
||||||
|
cleanPath = "/index"
|
||||||
|
}
|
||||||
|
dir := path.Dir(cleanPath)
|
||||||
|
base := path.Base(cleanPath)
|
||||||
|
if base == "." || base == "/" || base == "" {
|
||||||
|
base = "index"
|
||||||
|
}
|
||||||
|
ext := strings.ToLower(path.Ext(base))
|
||||||
|
name := strings.TrimSuffix(base, ext)
|
||||||
|
name = sanitizePathSegment(name)
|
||||||
|
if name == "" {
|
||||||
|
name = "image"
|
||||||
|
}
|
||||||
|
|
||||||
|
var segments []string
|
||||||
|
if dir != "." && dir != "/" {
|
||||||
|
for _, seg := range strings.Split(strings.TrimPrefix(dir, "/"), "/") {
|
||||||
|
if seg == "" || seg == "." || seg == ".." {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cleanSeg := sanitizePathSegment(seg)
|
||||||
|
if cleanSeg == "" {
|
||||||
|
cleanSeg = "segment"
|
||||||
|
}
|
||||||
|
segments = append(segments, cleanSeg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return segments, name, ext
|
||||||
|
}
|
||||||
|
|
||||||
|
func querySuffix(rawQuery string) string {
|
||||||
|
rawQuery = strings.TrimSpace(rawQuery)
|
||||||
|
if rawQuery == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(rawQuery))
|
||||||
|
return "__q_" + hex.EncodeToString(sum[:8])
|
||||||
|
}
|
||||||
|
|
||||||
func imageExtension(contentType, requestPath string) string {
|
func imageExtension(contentType, requestPath string) string {
|
||||||
if ext := strings.ToLower(filepath.Ext(requestPath)); ext != "" && ext != "." {
|
if ext := strings.ToLower(path.Ext(requestPath)); ext != "" && ext != "." {
|
||||||
return ext
|
return ext
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,3 +249,25 @@ func sanitizeName(s string) string {
|
|||||||
}
|
}
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sanitizePathSegment(s string) string {
|
||||||
|
s = strings.ToLower(strings.TrimSpace(s))
|
||||||
|
if s == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z':
|
||||||
|
b.WriteRune(r)
|
||||||
|
case r >= '0' && r <= '9':
|
||||||
|
b.WriteRune(r)
|
||||||
|
case r == '.', r == '-', r == '_':
|
||||||
|
b.WriteRune(r)
|
||||||
|
default:
|
||||||
|
b.WriteByte('_')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Trim(b.String(), "_")
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,7 +40,10 @@ func Run(cfg config.Config) error {
|
|||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
if !domain.IsMckoDomain(resp.Request.Host) {
|
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
|
||||||
|
isImage := capture.IsImageResponse(contentType, resp.Request.URL.Path)
|
||||||
|
shouldRewrite := domain.IsMckoDomain(resp.Request.Host) && rewrite.IsHTMLResponse(contentType)
|
||||||
|
if !isImage && !shouldRewrite {
|
||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,8 +54,7 @@ func Run(cfg config.Config) error {
|
|||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
|
if isImage {
|
||||||
if capture.IsImageResponse(contentType, resp.Request.URL.Path) {
|
|
||||||
if err := imageSaver.Save(resp, original); err != nil {
|
if err := imageSaver.Save(resp, original); err != nil {
|
||||||
log.Printf("save image failed for %s: %v", resp.Request.URL.String(), err)
|
log.Printf("save image failed for %s: %v", resp.Request.URL.String(), err)
|
||||||
}
|
}
|
||||||
@@ -60,7 +62,7 @@ func Run(cfg config.Config) error {
|
|||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rewrite.IsHTMLResponse(contentType) {
|
if !shouldRewrite {
|
||||||
setResponseBody(resp, original)
|
setResponseBody(resp, original)
|
||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,18 +27,27 @@ type Processor struct {
|
|||||||
func NewProcessor(hook HTMLHook) *Processor {
|
func NewProcessor(hook HTMLHook) *Processor {
|
||||||
return &Processor{hook: hook}
|
return &Processor{hook: hook}
|
||||||
}
|
}
|
||||||
|
func tryExtractLinks(text string) []string {
|
||||||
|
aiResponce, err := openrouter.AskOpenRouter(fmt.Sprintf("Ты - умный парсер для веб страниц. Данная страница представляет из себя тестовое задание и тебе нужно проанализировать страницу, найти блок в котором находится задание, и если внутри этого блока есть изображения - достать их и вернуть в виде JSON массива со строками на эти изображения. ВОЗВРАЩАЙ ТОЛЬКО И ТОЛЬКО JSON обьект, БЕЗ КАКИХ ЛИБО ДОПОЛНИТЕЛЬНЫХ СЛОВ, ФРАЗ, СИМВОЛОВ etc. В ФОРМАТЕ {\"images\":[\"test", "test\"]}", string(text)))
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("openrouter failed %s", err)
|
||||||
|
return []string{}
|
||||||
|
}
|
||||||
|
log.Println(aiResponce)
|
||||||
|
return []string{}
|
||||||
|
}
|
||||||
|
|
||||||
func DefaultQuestionTestHook(_ *http.Request, html []byte) ([]byte, error) {
|
func DefaultQuestionTestHook(_ *http.Request, html []byte) ([]byte, error) {
|
||||||
switch cheating.DetermineQuestionType(string(html)) {
|
switch cheating.DetermineQuestionType(string(html)) {
|
||||||
case cheating.FullTextQuestion:
|
case cheating.FullTextQuestion:
|
||||||
{
|
{
|
||||||
|
//log.Println(tryExtractLinks(string(html)))
|
||||||
aiResponce, err := openrouter.AskOpenRouter(fmt.Sprintf("РЕШИ ЗАДАНИЕ И ДАЙ МАКСИМАЛЬНО КРАТКИЙ ОТВЕТ: %s", string(html)))
|
aiResponce, err := openrouter.AskOpenRouter(fmt.Sprintf("прочитай задание и выпиши достаточно подробный список формул/гайд по решению которые могут помочь при решении задач ИЛИ ЕСЛИ МОЖЕШЬ СРАЗУ ДАТЬ ОТВЕТ - ответы списком: %s. пиши без latex обычным читаемым текстом и максимально кратко. учти что это работа за 10 класс российской школы", string(html)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("openrouter failed %s", err)
|
log.Printf("openrouter failed %s", err)
|
||||||
return html, nil
|
return html, nil
|
||||||
}
|
}
|
||||||
return []byte(strings.ReplaceAll(string(html), "generated", latex.ConvertLaTeXToASCII(aiResponce))), nil
|
return []byte(strings.ReplaceAll(string(html), "generated", fmt.Sprintf("<span title=\"%s\">generated</span>", latex.ConvertLaTeXToASCII(aiResponce)))), nil
|
||||||
}
|
}
|
||||||
case cheating.QuestionWithPicture:
|
case cheating.QuestionWithPicture:
|
||||||
{
|
{
|
||||||
|
|||||||
5
main.go
5
main.go
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"log"
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
"git.gulenok.ru/greenhaze/gaslight/internal/config"
|
"git.gulenok.ru/greenhaze/gaslight/internal/config"
|
||||||
"git.gulenok.ru/greenhaze/gaslight/internal/openrouter"
|
"git.gulenok.ru/greenhaze/gaslight/internal/openrouter"
|
||||||
@@ -9,8 +10,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
openrouter.Orclient.APIKey = "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"
|
openrouter.Orclient.APIKey = os.Getenv("OPENROUTER_TOKEN")
|
||||||
openrouter.Orclient.Model = "openai/gpt-5.4-nano"
|
openrouter.Orclient.Model = "openai/gpt-5.5"
|
||||||
cfg := config.ParseFlags()
|
cfg := config.ParseFlags()
|
||||||
if err := proxy.Run(cfg); err != nil {
|
if err := proxy.Run(cfg); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
|||||||
Reference in New Issue
Block a user