schedule_converter/parser/encoding.go

package parser

import (
	"bytes"
	"io"
	"net/http"
	"strings"

	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
)

func decodeResponseBody(resp *http.Response) (string, error) {
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return decodeBody(body, resp.Header.Get("Content-Type")), nil
}

func decodeBody(body []byte, contentType string) string {
	lowerCT := strings.ToLower(contentType)
	if strings.Contains(lowerCT, "charset=gb") || strings.Contains(lowerCT, "charset=gb2312") || strings.Contains(lowerCT, "charset=gbk") {
		decoded, err := decodeGBK(body)
		if err == nil {
			return decoded
		}
	}

	if isValidUTF8(body) {
		return string(body)
	}

	decoded, err := decodeGBK(body)
	if err == nil {
		return decoded
	}
	return string(body)
}

func decodeGBK(data []byte) (string, error) {
	reader := transform.NewReader(bytes.NewReader(data), simplifiedchinese.GBK.NewDecoder())
	decoded, err := io.ReadAll(reader)
	if err != nil {
		return "", err
	}
	return string(decoded), nil
}

func isValidUTF8(data []byte) bool {
	for i := 0; i < len(data); i++ {
		if data[i]&0x80 != 0 {
			if data[i]&0xE0 == 0xC0 {
				if i+1 >= len(data) || data[i+1]&0xC0 != 0x80 {
					return false
				}
				i++
			} else if data[i]&0xF0 == 0xE0 {
				if i+2 >= len(data) || data[i+1]&0xC0 != 0x80 || data[i+2]&0xC0 != 0x80 {
					return false
				}
				i += 2
			} else if data[i]&0xF8 == 0xF0 {
				if i+3 >= len(data) || data[i+1]&0xC0 != 0x80 || data[i+2]&0xC0 != 0x80 || data[i+3]&0xC0 != 0x80 {
					return false
				}
				i += 3
			} else {
				return false
			}
		}
	}
	return true
}