package main
import (
"fmt"
"net"
"os"
"strconv"
"strings"
"time"
)
// Config is the process configuration, read once from the environment.
type Config struct {
DataDir string
HTTPAddr string
SSHAddr string
PublicURL string
SSHHost string
SSHClientPort int
Quota int64
TokenTTL time.Duration
TokenSecret string
GitHubBase string
KeysTTL time.Duration
KeysInterval time.Duration
RRsyncPath string
MonitorEvery time.Duration
MaxEntries int
MaxDepth int
MaxUploadFiles int
}
// LoadConfig reads the environment and applies defaults suitable for a
// single-container deployment.
func LoadConfig() (Config, error) {
cfg := Config{
DataDir: envString("CDN_DATA", "/data"),
HTTPAddr: envString("CDN_HTTP_ADDR", ":8080"),
SSHAddr: envString("CDN_SSH_ADDR", ":2222"),
PublicURL: strings.TrimRight(envString("CDN_PUBLIC_URL", "https://cdn.si"), "/"),
SSHHost: envString("CDN_SSH_HOST", "cdn.si"),
Quota: envInt64("CDN_QUOTA_BYTES", 50<<20),
TokenTTL: envDuration("CDN_TOKEN_TTL", 15*time.Minute),
TokenSecret: os.Getenv("CDN_TOKEN_SECRET"),
GitHubBase: strings.TrimRight(envString("CDN_GITHUB_BASE", "https://github.com"), "/"),
KeysTTL: envDuration("CDN_KEYS_TTL", 5*time.Minute),
KeysInterval: envDuration("CDN_KEYS_FETCH_INTERVAL", 30*time.Second),
RRsyncPath: envString("CDN_RRSYNC", "/usr/bin/rrsync"),
MonitorEvery: envDuration("CDN_MONITOR_INTERVAL", 100*time.Millisecond),
MaxEntries: int(envInt64("CDN_MAX_ENTRIES", 10000)),
MaxDepth: int(envInt64("CDN_MAX_DEPTH", 32)),
MaxUploadFiles: int(envInt64("CDN_MAX_UPLOAD_FILES", 100)),
}
if port := os.Getenv("CDN_SSH_PORT"); port != "" {
value, err := strconv.Atoi(port)
if err != nil || value < 1 || value > 65535 {
return Config{}, fmt.Errorf("CDN_SSH_PORT: %q is not a port", port)
}
cfg.SSHClientPort = value
}
if cfg.Quota <= 0 {
return Config{}, fmt.Errorf("CDN_QUOTA_BYTES must be positive")
}
if cfg.SSHHost == "" {
return Config{}, fmt.Errorf("CDN_SSH_HOST must not be empty")
}
if cfg.MaxEntries <= 0 || cfg.MaxDepth <= 0 || cfg.MaxUploadFiles <= 0 {
return Config{}, fmt.Errorf("entry, depth and upload limits must be positive")
}
if cfg.MonitorEvery <= 0 {
return Config{}, fmt.Errorf("CDN_MONITOR_INTERVAL must be positive")
}
if cfg.TokenTTL <= 0 {
return Config{}, fmt.Errorf("CDN_TOKEN_TTL must be positive")
}
if cfg.KeysTTL <= 0 || cfg.KeysInterval <= 0 {
return Config{}, fmt.Errorf("CDN_KEYS_TTL and CDN_KEYS_FETCH_INTERVAL must be positive")
}
return cfg, nil
}
// SSHPort is the port shown to users, not necessarily the listener port.
func (c Config) SSHPort() int {
if c.SSHClientPort > 0 {
return c.SSHClientPort
}
_, port, err := net.SplitHostPort(c.SSHAddr)
if err != nil {
return 22
}
value, err := strconv.Atoi(port)
if err != nil {
return 22
}
return value
}
// SSHCommand is the ssh prefix shown in the UI and used by CI examples.
func (c Config) SSHCommand(user string) string {
if port := c.SSHPort(); port != 22 {
return fmt.Sprintf("ssh -p %d %s@%s", port, user, c.SSHHost)
}
return fmt.Sprintf("ssh %s@%s", user, c.SSHHost)
}
// SSHEOption is the rsync -e argument, empty for the default SSH port.
func (c Config) SSHEOption() string {
if port := c.SSHPort(); port != 22 {
return fmt.Sprintf("-e 'ssh -p %d'", port)
}
return ""
}
// RSyncCommand renders the publish command for one project folder.
func (c Config) RSyncCommand(user, project string) string {
parts := []string{"rsync", "-a", "--delete"}
if option := c.SSHEOption(); option != "" {
parts = append(parts, option)
}
parts = append(parts, "./", fmt.Sprintf("%s@%s:/%s/", user, c.SSHHost, project))
return strings.Join(parts, " ")
}
func envString(key, fallback string) string {
if value := os.Getenv(key); value != "" {
return value
}
return fallback
}
func envInt64(key string, fallback int64) int64 {
value := os.Getenv(key)
if value == "" {
return fallback
}
parsed, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return fallback
}
return parsed
}
func envDuration(key string, fallback time.Duration) time.Duration {
value := os.Getenv(key)
if value == "" {
return fallback
}
parsed, err := time.ParseDuration(value)
if err != nil {
return fallback
}
return parsed
}
package main
import (
"sort"
"strings"
"time"
)
// Entry is one row in a directory listing or breadcrumb.
type Entry struct {
Name string
Href string
Dir bool
Link bool
Size int64
ModTime time.Time
}
// SortEntries puts directories first, then names case-insensitively.
func SortEntries(entries []Entry) {
sort.Slice(entries, func(i, j int) bool {
if entries[i].Dir != entries[j].Dir {
return entries[i].Dir
}
return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name)
})
}
package main
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// errNoGitHubUser means GitHub does not know the account.
var errNoGitHubUser = errors.New("github user not found")
// GitHub fetches the public SSH keys a GitHub user has published.
type GitHub struct {
base string
client *http.Client
}
// NewGitHub returns a client for the configured GitHub base URL.
func NewGitHub(base string) *GitHub {
return &GitHub{
base: base,
client: &http.Client{
Timeout: 10 * time.Second,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 3 {
return fmt.Errorf("too many redirects")
}
if req.URL.Host != via[0].URL.Host {
return fmt.Errorf("redirect leaves %s", via[0].URL.Host)
}
return nil
},
},
}
}
// Keys returns the raw github.com/<user>.keys document, checking that at
// least one line parses as an SSH public key.
func (g *GitHub) Keys(ctx context.Context, user string) ([]byte, error) {
// The username is the path segment and the identity claim; only the
// GitHub alphabet may reach the URL.
user, err := NormalizeUser(user)
if err != nil {
return nil, err
}
url := g.base + "/" + user + ".keys"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "uploadservice")
resp, err := g.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, errNoGitHubUser
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("github keys: %s", resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
if err != nil {
return nil, err
}
if len(body) > 1<<20 {
return nil, fmt.Errorf("github keys: response too large")
}
keys, err := ParseAuthorizedKeys(body)
if err != nil {
return nil, err
}
if len(keys) == 0 {
return nil, fmt.Errorf("github keys: no usable public keys")
}
if !strings.HasSuffix(string(body), "\n") {
body = append(body, '\n')
}
return body, nil
}
// Copyright 2022 Fortio Authors
// Package assert is a minimalistic drop-in replacement for testify/assert.
// It derives from fortio.org/assert and is internal to this module.
package assert
import (
"fmt"
"reflect"
"regexp"
"runtime"
"strings"
"testing"
)
// ObjectsAreEqualValues returns true if a == b (through refection).
func ObjectsAreEqualValues(a, b any) bool {
return reflect.DeepEqual(a, b)
}
// Errorf is a local variant to get the right line numbers.
func Errorf(t testing.TB, format string, rest ...any) {
_, file, line, _ := runtime.Caller(2)
file = file[strings.LastIndex(file, "/")+1:]
fmt.Printf("%s:%d %s\n", file, line, fmt.Sprintf(format, rest...))
t.Fail()
}
// NotEqual checks for a not equal b.
func NotEqual(t testing.TB, a, b any, msg ...any) {
if ObjectsAreEqualValues(a, b) {
Errorf(t, "%v unexpectedly equal: %v", a, msg)
}
}
// EqualValues checks for a equal b.
func EqualValues(t testing.TB, a, b any, msg ...any) {
if !ObjectsAreEqualValues(a, b) {
Errorf(t, "%v unexpectedly not equal %v: %v", a, b, msg)
}
}
// Equal also checks for a equal b.
func Equal(t testing.TB, a, b any, msg ...any) {
if !ObjectsAreEqualValues(a, b) {
Errorf(t, "%v unexpectedly not equal %v: %v", a, b, msg)
}
}
// NoError checks for no errors (nil).
func NoError(t testing.TB, err error, msg ...any) {
if err != nil {
Errorf(t, "expecting no error, got %v: %v", err, msg)
}
}
// Error checks/expects an error.
func Error(t testing.TB, err error, msg ...any) {
if err == nil {
Errorf(t, "expecting an error, didn't get it: %v", msg)
}
}
// True checks bool is true.
func True(t testing.TB, b bool, msg ...any) {
if !b {
Errorf(t, "expecting true, didn't: %v", msg)
}
}
// False checks bool is false.
func False(t testing.TB, b bool, msg ...any) {
if b {
Errorf(t, "expecting false, didn't: %v", msg)
}
}
// Contains checks that needle is in haystack.
func Contains(t testing.TB, haystack, needle any, msg ...any) {
if !containsElement(haystack, needle) {
Errorf(t, "%v doesn't contain %v: %v", haystack, needle, msg)
}
}
// Fail fails the test.
func Fail(t testing.TB, msg string, args ...any) {
_, file, line, _ := runtime.Caller(1)
file = file[strings.LastIndex(file, "/")+1:]
if len(args) > 0 {
msg = fmt.Sprintf("%s: %v", msg, args)
}
fmt.Printf("%s:%d %s\n", file, line, msg)
t.FailNow()
}
// CheckEquals fails the test, logging filename:linenum, when actual and expected differ.
func CheckEquals(t testing.TB, actual, expected any, msg ...any) {
if expected != actual {
_, file, line, _ := runtime.Caller(1)
file = file[strings.LastIndex(file, "/")+1:]
fmt.Printf("%s:%d mismatch!\nactual:\n%+v\nexpected:\n%+v\nfor %+v\n", file, line, actual, expected, msg)
t.Fail()
}
}
// Assert is True under the name of the earlier fortio/stats tests.
func Assert(t testing.TB, cond bool, msg string, rest ...any) {
if !cond {
_, file, line, _ := runtime.Caller(1)
file = file[strings.LastIndex(file, "/")+1:]
m := fmt.Sprintf(msg, rest...)
fmt.Printf("%s:%d assert failure: %s\n", file, line, m)
t.Fail()
}
}
type hasT interface {
T() *testing.T
SetT(t *testing.T)
}
// TestSuite is the base struct for test suites.
// Replaces https://pkg.go.dev/github.com/stretchr/testify@v1.8.0/suite.
type TestSuite struct {
t *testing.T
}
// T returns the current testing.T.
func (s *TestSuite) T() *testing.T {
return s.t
}
// SetT sets the testing.T in the suite object.
func (s *TestSuite) SetT(t *testing.T) {
s.t = t
}
type hasSetupTest interface {
SetupTest()
}
type hasTearDown interface {
TearDownTest()
}
// Run runs the suite with SetupTest first and TearDownTest after, as testify's suite.Run does.
func Run(t *testing.T, suite hasT) {
suite.SetT(t)
tests := []testing.InternalTest{}
methodFinder := reflect.TypeOf(suite)
var setup hasSetupTest
if s, ok := suite.(hasSetupTest); ok {
setup = s
}
var tearDown hasTearDown
if td, ok := suite.(hasTearDown); ok {
tearDown = td
}
for method := range methodFinder.Methods() {
//nolint:staticcheck // consider fixing later for perf but this is just to run a few tests.
if ok, _ := regexp.MatchString("^Test", method.Name); !ok {
continue
}
test := testing.InternalTest{
Name: method.Name,
F: func(_ *testing.T) {
method.Func.Call([]reflect.Value{reflect.ValueOf(suite)})
},
}
tests = append(tests, test)
}
for _, test := range tests {
if setup != nil {
setup.SetupTest()
}
t.Run(test.Name, test.F)
if tearDown != nil {
tearDown.TearDownTest()
}
}
}
// Copyright 2022 Fortio Authors
// Licensed under the Apache License, Version 2.0.
package assert
import (
"errors"
"reflect"
"strings"
"testing"
)
// Nil checks that value is nil, including a typed nil.
func Nil(t testing.TB, value any, msg ...any) {
if !isNil(value) {
Errorf(t, "expecting nil, got %v: %v", value, msg)
}
}
// NotNil checks that value is not nil, including a typed nil.
func NotNil(t testing.TB, value any, msg ...any) {
if isNil(value) {
Errorf(t, "expecting non-nil: %v", msg)
}
}
// Len checks the length of a string, array, channel, map, or slice.
func Len(t testing.TB, value any, expected int, msg ...any) bool {
v := reflect.ValueOf(value)
if !hasLength(v) {
Errorf(t, "%T has no length: %v", value, msg)
return false
}
if actual := v.Len(); actual != expected {
Errorf(t, "length %d unexpectedly not equal %d: %v", actual, expected, msg)
return false
}
return true
}
// Empty checks that value is nil, zero, or has length zero.
func Empty(t testing.TB, value any, msg ...any) {
if !isEmpty(value) {
Errorf(t, "expecting empty, got %v: %v", value, msg)
}
}
// NotEmpty checks that value is neither nil nor empty.
func NotEmpty(t testing.TB, value any, msg ...any) {
if isEmpty(value) {
Errorf(t, "expecting non-empty: %v", msg)
}
}
// Greater checks that actual is greater than expected.
func Greater(t testing.TB, actual, expected any, msg ...any) {
if cmp, ok := ordered(actual, expected); !ok || cmp <= 0 {
Errorf(t, "%v is not greater than %v: %v", actual, expected, msg)
}
}
// NotContains checks that needle is not in haystack.
func NotContains(t testing.TB, haystack, needle any, msg ...any) {
if containsElement(haystack, needle) {
Errorf(t, "%v unexpectedly contains %v: %v", haystack, needle, msg)
}
}
// ErrorContains checks that err is non-nil and its message contains want.
func ErrorContains(t testing.TB, err error, want string, msg ...any) {
if err == nil {
Errorf(t, "expecting an error containing %q, got nil: %v", want, msg)
return
}
if !strings.Contains(err.Error(), want) {
Errorf(t, "%v does not contain %q: %v", err, want, msg)
}
}
// ErrorIs checks that err matches target.
func ErrorIs(t testing.TB, err, target error, msg ...any) {
if !errors.Is(err, target) {
Errorf(t, "%v does not match %v: %v", err, target, msg)
}
}
// IsIncreasing checks that every item is greater than the item before it.
func IsIncreasing(t testing.TB, value any, msg ...any) {
v := reflect.ValueOf(value)
if v.Kind() != reflect.Array && v.Kind() != reflect.Slice {
Errorf(t, "%T is not an array or slice: %v", value, msg)
return
}
for i := 1; i < v.Len(); i++ {
if cmp, ok := ordered(v.Index(i).Interface(), v.Index(i-1).Interface()); !ok || cmp <= 0 {
Errorf(t, "%v is not increasing at index %d: %v", value, i, msg)
return
}
}
}
func isNil(value any) bool {
if value == nil {
return true
}
v := reflect.ValueOf(value)
switch v.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return v.IsNil()
default:
return false
}
}
func isEmpty(value any) bool {
if value == nil {
return true
}
v := reflect.ValueOf(value)
if hasLength(v) {
return v.Len() == 0
}
return v.IsZero()
}
func hasLength(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String:
return true
default:
return false
}
}
func containsElement(haystack, needle any) bool {
v := reflect.ValueOf(haystack)
if !v.IsValid() {
return false
}
switch v.Kind() {
case reflect.String:
s, ok := needle.(string)
return ok && strings.Contains(v.String(), s)
case reflect.Array, reflect.Slice:
for i := 0; i < v.Len(); i++ {
if reflect.DeepEqual(v.Index(i).Interface(), needle) {
return true
}
}
case reflect.Map:
key := reflect.ValueOf(needle)
return key.IsValid() && key.Type().AssignableTo(v.Type().Key()) && v.MapIndex(key).IsValid()
}
return false
}
// ordered compares two values of the same broad kind. The second return
// is false when the two are not comparable, which every caller has to
// treat as a failure: a bare 0 would read as "equal" and let
// LessOrEqual and GreaterOrEqual pass on an int against a string.
func ordered(actual, expected any) (int, bool) {
a := reflect.ValueOf(actual)
b := reflect.ValueOf(expected)
if !a.IsValid() || !b.IsValid() {
return 0, false
}
switch a.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if b.Kind() >= reflect.Int && b.Kind() <= reflect.Int64 {
return compare(a.Int(), b.Int()), true
}
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if b.Kind() >= reflect.Uint && b.Kind() <= reflect.Uintptr {
return compare(a.Uint(), b.Uint()), true
}
case reflect.Float32, reflect.Float64:
if b.Kind() == reflect.Float32 || b.Kind() == reflect.Float64 {
return compare(a.Float(), b.Float()), true
}
case reflect.String:
if b.Kind() == reflect.String {
return strings.Compare(a.String(), b.String()), true
}
}
return 0, false
}
func compare[T ~int64 | ~uint64 | ~float64](a, b T) int {
if a < b {
return -1
}
if a > b {
return 1
}
return 0
}
package markdown
import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
chroma "github.com/alecthomas/chroma/v2"
"github.com/alecthomas/chroma/v2/formatters/html"
"github.com/alecthomas/chroma/v2/lexers"
"github.com/alecthomas/chroma/v2/styles"
blackfriday "github.com/russross/blackfriday/v2"
)
// Renderer renders markdown content to HTML with syntax highlighting.
type Renderer struct {
// codeBlockPattern matches HTML code blocks for highlighting
codeBlockPattern *regexp.Regexp
}
// NewRenderer creates a new markdown renderer with syntax highlighting support.
func NewRenderer() *Renderer {
return &Renderer{
codeBlockPattern: regexp.MustCompile(`<pre><code(?:\s+class="language-([^"]+)")?>([.\s\S]*?)</code></pre>`),
}
}
// NormalizeNewlines folds CRLF and lone CR line endings into LF.
//
// This is not cosmetic. Both the HTML form spec and the FormData serializer
// normalize a textarea's value to CRLF on submit, so every article written in
// a browser arrives with CRLF. Blackfriday matches its block constructs
// against lines split on LF, which leaves a trailing CR on each one: the GFM
// table delimiter row stops matching, a closing code fence stops closing, and
// the whole block collapses into one paragraph. Folding the endings first is
// what makes a pasted table survive the round trip.
func NormalizeNewlines(content []byte) []byte {
if !bytes.ContainsRune(content, '\r') {
return content
}
content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))
return bytes.ReplaceAll(content, []byte("\r"), []byte("\n"))
}
// mediaExtensions names the files a browser can play, by the player each
// one needs. mdedit draws the same links as players while the article is
// written, and this is the same list: what the author hears is what the
// reader hears.
var mediaExtensions = map[string]string{
"mp3": "audio", "m4a": "audio", "aac": "audio", "oga": "audio", "ogg": "audio",
"opus": "audio", "wav": "audio", "flac": "audio", "weba": "audio",
"mp4": "video", "m4v": "video", "webm": "video", "ogv": "video", "mov": "video",
}
// MediaKind answers "audio", "video", or an empty string for an address that
// names neither. A query or a fragment on the end names nothing about the
// file, so neither is read.
func MediaKind(href string) string {
address := href
if cut := strings.IndexAny(address, "?#"); cut >= 0 {
address = address[:cut]
}
dot := strings.LastIndex(address, ".")
if dot < 0 {
return ""
}
return mediaExtensions[strings.ToLower(address[dot+1:])]
}
// anchorPattern matches one rendered link. markupPattern strips the markup
// out of a link's text, which may carry emphasis the author wrote: an
// attribute holds characters, not elements.
var (
anchorPattern = regexp.MustCompile(`<a href="([^"]*)"[^>]*>(.*?)</a>`)
markupPattern = regexp.MustCompile(`<[^>]*>`)
)
// players turns a link to a recording or a capture into the player for it.
// The element loads its length and no more until the reader plays it, so an
// article carrying an hour of audio costs a reader who scrolls past one
// short request.
func players(content []byte) []byte {
// Nothing to scan, and nothing to return but what came in: a document
// with no links at all is most of them.
if !bytes.Contains(content, []byte("<a href=")) {
return content
}
return anchorPattern.ReplaceAllFunc(content, func(match []byte) []byte {
parts := anchorPattern.FindSubmatch(match)
kind := MediaKind(string(parts[1]))
if kind == "" {
return match
}
label := strings.TrimSpace(string(markupPattern.ReplaceAll(parts[2], nil)))
label = strings.ReplaceAll(label, `"`, """)
return []byte(fmt.Sprintf(
`<%s class="media media--%s" controls preload="metadata" src="%s" title="%s" aria-label="%s"></%s>`,
kind, kind, parts[1], label, label, kind,
))
})
}
// runPattern matches a stretch of paragraphs holding one picture each, which
// is what a markdown body looks like when somebody has pasted a roll of
// photographs: `` on a line, a blank line, the next one.
var runPattern = regexp.MustCompile(`(?:<p>\s*<img [^>]*>\s*</p>\s*){3,}`)
// imgPattern picks the pictures back out of such a run.
var imgPattern = regexp.MustCompile(`<img [^>]*>`)
// SlideshowRun is how many pictures in a row become a slideshow. Two
// pictures are a comparison and read better side by side in the page; a
// roll of them is a thing to step through.
const SlideshowRun = 3
// Slideshow turns each run of pictures into one frame the reader steps
// through. The article requires it; a body that holds no such run is
// returned as it is.
//
// The markup is plain: a list of figures with one shown at a time, the
// buttons beside it, and a counter. Without JavaScript the frame is a
// column of pictures with a scrollbar, the column the body already held.
func Slideshow(content []byte) []byte {
if !bytes.Contains(content, []byte("<img ")) {
return content
}
show := 0
return runPattern.ReplaceAllFunc(content, func(run []byte) []byte {
pictures := imgPattern.FindAll(run, -1)
if len(pictures) < SlideshowRun {
return run
}
show++
var out bytes.Buffer
fmt.Fprintf(&out, `<div class="slideshow" data-slideshow="%d" aria-roledescription="slideshow">`, show)
fmt.Fprintf(&out, `<div class="slideshow__frame">`)
for index, picture := range pictures {
fmt.Fprintf(&out,
`<figure class="slideshow__slide" role="group" aria-roledescription="slide" aria-label="%d of %d"%s>%s</figure>`,
index+1, len(pictures), map[bool]string{true: "", false: ` hidden`}[index == 0], picture,
)
}
out.WriteString(`</div>`)
fmt.Fprintf(&out,
`<div class="slideshow__bar"><button type="button" class="slideshow__step" data-step="-1" aria-label="Previous picture">←</button>`+
`<span class="slideshow__count" aria-live="polite">1 / %d</span>`+
`<button type="button" class="slideshow__step" data-step="1" aria-label="Next picture">→</button></div>`,
len(pictures),
)
out.WriteString(`</div>`)
return out.Bytes()
})
}
// EmbedPattern matches the <!-- article#id --> marker a body uses to cite
// another article.
var EmbedPattern = regexp.MustCompile(`<!--\s*article#([0-9A-HJKMNP-TV-Z]{26})\s*-->`)
// Marker is one embed marker found in a markdown body, outside any fenced
// code block.
type Marker struct {
Start int
End int
ID string
}
// FindMarkers answers the embed markers a body carries, skipping any that sit
// inside a ``` or ~~~ fenced code block: a marker in a fence is code to show,
// not a citation, so it renders literally and records nothing. The renderer
// and the storage reconcile both read this list, which keeps the cards on the
// page and the article_embed rows the same set. Content should already be
// LF-normalized; both callers are.
func FindMarkers(content []byte) []Marker {
matches := EmbedPattern.FindAllSubmatchIndex(content, -1)
if len(matches) == 0 {
return nil
}
fences := fencedRanges(content)
var markers []Marker
for _, m := range matches {
if intersects(fences, m[0], m[1]) {
continue
}
markers = append(markers, Marker{Start: m[0], End: m[1], ID: string(content[m[2]:m[3]])})
}
return markers
}
// byteRange is one half-open [start, end) span of the content.
type byteRange struct {
start int
end int
}
// intersects reports whether [start, end) overlaps any of the ranges. A
// marker that so much as touches a fence is left alone: splitting it out
// would destroy the fence around it.
func intersects(ranges []byteRange, start, end int) bool {
for _, r := range ranges {
if start < r.end && end > r.start {
return true
}
}
return false
}
// fencedRanges scans each line of the content and answers the byte ranges
// covered by fenced code blocks, delimiter lines included. The rules follow
// what blackfriday accepts: three or more backticks or tildes after at most
// three spaces of indent open a fence, a backtick fence's language tag may
// not carry another backtick, the closer is at least as long in the same
// character with nothing else on the line, and an unclosed fence runs to the
// end of the document.
func fencedRanges(content []byte) []byteRange {
var ranges []byteRange
var fenceChar byte
fenceLen, fenceStart := 0, 0
at := 0
for at < len(content) {
end := bytes.IndexByte(content[at:], '\n')
if end < 0 {
end = len(content)
} else {
end += at + 1
}
line := bytes.TrimSuffix(content[at:end], []byte("\n"))
if char, length, rest, ok := fenceLine(line); ok {
switch {
case fenceChar == 0:
if char != '`' || !bytes.ContainsRune(rest, '`') {
fenceChar, fenceLen, fenceStart = char, length, at
}
case char == fenceChar && length >= fenceLen && len(bytes.TrimSpace(rest)) == 0:
ranges = append(ranges, byteRange{start: fenceStart, end: end})
fenceChar = 0
}
}
at = end
}
if fenceChar != 0 {
ranges = append(ranges, byteRange{start: fenceStart, end: len(content)})
}
return ranges
}
// fenceLine reports whether a line is a fence delimiter: at most three spaces
// of indent, then a run of three or more backticks or tildes. It answers the
// fence character, the run length and whatever follows the run.
func fenceLine(line []byte) (char byte, length int, rest []byte, ok bool) {
indent := 0
for indent < len(line) && line[indent] == ' ' {
indent++
}
if indent > 3 || indent == len(line) {
return 0, 0, nil, false
}
char = line[indent]
if char != '`' && char != '~' {
return 0, 0, nil, false
}
length = 0
for indent+length < len(line) && line[indent+length] == char {
length++
}
if length < 3 {
return 0, 0, nil, false
}
return char, length, line[indent+length:], true
}
// RenderEmbedded renders content with each embed marker replaced by the card
// HTML its id maps to. The marker cannot survive blackfriday: on its own line
// it parses as an HTMLBlock, and SkipHTML drops that node without writing, so
// the source is split on the markers and each segment rendered on its own. An
// id with no card renders as nothing. A marker inside a fenced code block is
// not split out at all - the fence stays whole and blackfriday shows the
// marker as the code it is.
func (r *Renderer) RenderEmbedded(content []byte, cards map[string]string) []byte {
content = NormalizeNewlines(content)
markers := FindMarkers(content)
if len(markers) == 0 {
return r.Render(content)
}
var out bytes.Buffer
at := 0
for _, m := range markers {
out.Write(r.Render(content[at:m.Start]))
out.WriteString(cards[m.ID])
at = m.End
}
out.Write(r.Render(content[at:]))
return out.Bytes()
}
// Render converts markdown content to HTML with syntax highlighting for code blocks
// Security: raw inline HTML in the markdown is skipped (multi-user content is
// rendered unescaped by templates), and Safelink neutralizes javascript:,
// data: and vbscript: destinations - an unsafe link renders as text, while
// relative, http(s), ftp and mailto links stay links. Blackfriday escapes
// HTML in code blocks, we unescape to get raw code, then Chroma's formatter
// escapes it again when outputting HTML.
func (r *Renderer) Render(content []byte) []byte {
renderer := &guardedRenderer{
safe: blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
Flags: blackfriday.CommonHTMLFlags | blackfriday.SkipHTML,
}),
unsafe: blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
Flags: blackfriday.CommonHTMLFlags | blackfriday.SkipHTML | blackfriday.Safelink,
}),
}
parser := blackfriday.New(blackfriday.WithExtensions(blackfriday.CommonExtensions))
ast := parser.Parse(NormalizeNewlines(content))
var buf bytes.Buffer
renderer.RenderHeader(&buf, ast)
ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
return renderer.RenderNode(&buf, node, entering)
})
renderer.RenderFooter(&buf, ast)
htmlContent := buf.Bytes()
// Then apply syntax highlighting to code blocks
highlighted := r.highlightCodeBlocks(htmlContent)
// A link to a file a browser can play is a player. Code blocks are
// escaped by now, so a link shown as code stays the text it is.
return players(highlighted)
}
// schemePattern matches the prefix of a URI that names a scheme, such as
// "javascript:" or "https:". A destination with no scheme is a relative link,
// a query or a fragment.
var schemePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.-]*:`)
// linkScheme answers the lowercased scheme of a link destination, or an empty
// string for a destination that names none. Spaces and control characters
// come out first, because a browser drops them before it reads the scheme:
// "java\tscript:" is a script link with a tab in it.
func linkScheme(destination []byte) string {
cleaned := make([]byte, 0, len(destination))
for _, char := range destination {
if char > ' ' {
cleaned = append(cleaned, char)
}
}
match := schemePattern.Find(cleaned)
if match == nil {
return ""
}
return strings.ToLower(string(match[:len(match)-1]))
}
// linkAllowed reports whether a destination stays a link. A relative link,
// a query and a fragment read from the page's own site; http, https, ftp and
// mailto name the schemes a document may link to. Everything else, notably
// javascript, data and vbscript, is script in a link's clothing.
func linkAllowed(destination []byte) bool {
switch linkScheme(destination) {
case "", "http", "https", "ftp", "mailto":
return true
}
return false
}
// guardedRenderer sends every link whose destination is not allowed to the
// Safelink renderer, which draws its text and no anchor. Everything else goes
// to the plain renderer, so a relative destination or a fragment keeps its
// link: blackfriday's own Safelink accepts only "/", "./" and "../", which
// drops the plain `[next](usage.md)` and `[top](#head)` a README carries.
type guardedRenderer struct {
safe *blackfriday.HTMLRenderer
unsafe *blackfriday.HTMLRenderer
}
func (r *guardedRenderer) RenderHeader(w io.Writer, ast *blackfriday.Node) {
r.safe.RenderHeader(w, ast)
}
func (r *guardedRenderer) RenderFooter(w io.Writer, ast *blackfriday.Node) {
r.safe.RenderFooter(w, ast)
}
func (r *guardedRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
if node.Type == blackfriday.Link && !linkAllowed(node.Destination) {
return r.unsafe.RenderNode(w, node, entering)
}
return r.safe.RenderNode(w, node, entering)
}
// highlightCodeBlocks applies syntax highlighting to all code blocks in the HTML.
func (r *Renderer) highlightCodeBlocks(html []byte) []byte {
htmlStr := string(html)
// Replace all code blocks with highlighted versions
result := r.codeBlockPattern.ReplaceAllStringFunc(htmlStr, func(match string) string {
return r.processCodeBlock(match)
})
return []byte(result)
}
// processCodeBlock processes a single code block and applies syntax highlighting.
func (r *Renderer) processCodeBlock(match string) string {
// Extract language and code from the match
m := r.codeBlockPattern.FindStringSubmatch(match)
if len(m) < 3 {
return match
}
language := m[1]
code := m[2]
// Unescape HTML entities in code (blackfriday escapes it, we need original for highlighting)
code = unescapeHTML(code)
// Apply syntax highlighting - Chroma handles escaping internally
highlighted := highlightCode([]byte(code), language)
return string(highlighted)
}
// highlightCode applies syntax highlighting to code using Chroma.
func highlightCode(code []byte, language string) []byte {
if len(code) == 0 {
return []byte("<pre><code></code></pre>")
}
codeStr := string(code)
// Select lexer for the language
var lexer chroma.Lexer
if language != "" {
lexer = lexers.Get(language)
}
if lexer == nil {
lexer = lexers.Analyse(codeStr)
}
if lexer == nil {
lexer = lexers.Fallback
}
// Tokenize
iterator, err := lexer.Tokenise(nil, codeStr)
if err != nil {
// On error, return plain code block
return wrapCodePlain(code, language)
}
// Format with HTML and Monokai style
style := styles.Get("monokai")
if style == nil {
style = styles.Fallback
}
formatter := html.New(
html.WithClasses(false),
html.PreventSurroundingPre(true),
html.TabWidth(4),
)
var buf bytes.Buffer
if err := formatter.Format(&buf, style, iterator); err != nil {
return wrapCodePlain(code, language)
}
// Don't strip newlines - preserve all whitespace as-is
formattedCode := buf.String()
// Wrap highlighted code in pre/code tags with chroma and language classes
languageClass := ""
if language != "" {
languageClass = fmt.Sprintf(` language-%s`, language)
}
result := fmt.Sprintf(
"<pre class=\"chroma%s\"><code>%s</code></pre>",
languageClass,
formattedCode,
)
return []byte(result)
}
// wrapCodePlain wraps code without highlighting.
func wrapCodePlain(code []byte, language string) []byte {
languageClass := ""
if language != "" {
languageClass = fmt.Sprintf(` class="language-%s"`, language)
}
return []byte(fmt.Sprintf(
"<pre><code%s>%s</code></pre>",
languageClass,
escapeHTML(string(code)),
))
}
// escapeHTML escapes HTML special characters.
func escapeHTML(s string) string {
return strings.NewReplacer(
"&", "&",
"<", "<",
">", ">",
"\"", """,
"'", "'",
).Replace(s)
}
// unescapeHTML unescapes HTML special characters.
func unescapeHTML(s string) string {
return strings.NewReplacer(
"&", "&",
"<", "<",
">", ">",
""", "\"",
"'", "'",
).Replace(s)
}
package main
import (
"context"
"errors"
"log/slog"
"os"
"sync"
"time"
"golang.org/x/crypto/ssh"
)
// KeyCache authenticates SSH keys against the keys GitHub publishes for a
// user. The cached document is trusted for ttl; past that, or when the
// presented key is not in it, the document is fetched again. Fetches for one
// user are at least interval apart, so a wrong key cannot turn every login
// attempt into a GitHub request.
type KeyCache struct {
github *GitHub
store *Store
ttl time.Duration
interval time.Duration
now func() time.Time
mu sync.Mutex
attempts map[string]*keyAttempt
}
type keyAttempt struct {
mu sync.Mutex
last time.Time
}
// NewKeyCache returns a cache that reads GitHub through github and persists
// key documents in store.
func NewKeyCache(github *GitHub, store *Store, ttl, interval time.Duration) *KeyCache {
if ttl <= 0 {
ttl = 5 * time.Minute
}
if interval <= 0 {
interval = 30 * time.Second
}
return &KeyCache{
github: github,
store: store,
ttl: ttl,
interval: interval,
now: time.Now,
attempts: make(map[string]*keyAttempt),
}
}
// Verify reports whether GitHub publishes key for the normalised user. A
// GitHub outage falls back to the last cached document; a GitHub 404 does
// not, so a removed account stops authenticating.
func (c *KeyCache) Verify(ctx context.Context, user string, key ssh.PublicKey) bool {
if c.fresh(user) && c.store.KeyEqual(user, key) {
return true
}
switch err := c.refresh(ctx, user); {
case errors.Is(err, errNoGitHubUser):
return false
case err != nil:
slog.Warn("github keys", "user", user, "error", err)
}
return c.store.KeyEqual(user, key)
}
// fresh reports whether the cached document is inside its TTL.
func (c *KeyCache) fresh(user string) bool {
info, err := os.Stat(c.store.KeysPath(user))
if err != nil {
return false
}
return c.now().Sub(info.ModTime()) < c.ttl
}
// refresh fetches and stores the key document unless this user was fetched
// too recently. The fetch runs under the per-user attempt lock, so waiters
// observe the fresh document instead of repeating the request.
func (c *KeyCache) refresh(ctx context.Context, user string) error {
attempt := c.attemptFor(user)
attempt.mu.Lock()
defer attempt.mu.Unlock()
now := c.now()
if now.Sub(attempt.last) < c.interval {
return nil
}
attempt.last = now
fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
keys, err := c.github.Keys(fetchCtx, user)
if err != nil {
return err
}
return c.store.SaveKeys(user, keys)
}
func (c *KeyCache) attemptFor(user string) *keyAttempt {
c.mu.Lock()
defer c.mu.Unlock()
attempt, ok := c.attempts[user]
if !ok {
attempt = &keyAttempt{}
c.attempts[user] = attempt
}
return attempt
}
package main
import (
"html/template"
)
// logoSVG is the cdn.si mark: three stacked plates, the top one filled. It is
// the mark beside the wordmark and the favicon, so the tab and the page carry
// the same shape. currentColor follows --signal on a page; the fallback is the
// dark theme's accent, which is what a standalone favicon has to use.
const logoSVG = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round" stroke-linecap="round" style="color:var(--signal,#e0a44f)" aria-hidden="true"><path fill="currentColor" fill-opacity=".22" fill-rule="evenodd" d="M12 3.2 20.4 7.6 12 12 3.6 7.6Z M12 5.8 17.2 7.6 12 9.4 6.8 7.6Z"/><path d="M3.6 12 12 16.4 20.4 12"/><path d="M3.6 16.4 12 20.8 20.4 16.4" stroke-opacity=".5"/></svg>`
// mark returns the logo for a template.
func mark() template.HTML {
return template.HTML(logoSVG)
}
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
if err := run(); err != nil {
slog.Error("uploadservice", "error", err)
os.Exit(1)
}
}
func run() error {
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, nil)))
cfg, err := LoadConfig()
if err != nil {
return err
}
store := NewStore(cfg)
if err := store.Init(); err != nil {
return fmt.Errorf("data directory: %w", err)
}
secret, err := store.EnsureSecret(cfg.TokenSecret)
if err != nil {
return fmt.Errorf("token secret: %w", err)
}
if cfg.TokenSecret == "" {
slog.Info("token secret loaded from data directory; set CDN_TOKEN_SECRET to share tokens across replicas")
}
tokens := NewToken(secret, cfg.TokenTTL)
github := NewGitHub(cfg.GitHubBase)
web, err := NewServer(cfg, store, tokens, github)
if err != nil {
return err
}
sshService, err := NewSSHD(cfg, store, tokens, github)
if err != nil {
return fmt.Errorf("ssh server: %w", err)
}
httpListener, err := net.Listen("tcp", cfg.HTTPAddr)
if err != nil {
return fmt.Errorf("http listener: %w", err)
}
defer httpListener.Close()
sshListener, err := net.Listen("tcp", cfg.SSHAddr)
if err != nil {
return fmt.Errorf("ssh listener: %w", err)
}
defer sshListener.Close()
httpServer := &http.Server{
Handler: web.Handler(),
ReadHeaderTimeout: 10 * time.Second,
IdleTimeout: 60 * time.Second,
MaxHeaderBytes: 1 << 20,
}
slog.Info("listening",
"http", httpListener.Addr().String(),
"ssh", sshListener.Addr().String(),
"data", cfg.DataDir,
"quota", humanSize(cfg.Quota),
"public", cfg.PublicURL,
)
failures := make(chan error, 2)
go func() { failures <- httpServer.Serve(httpListener) }()
go func() { failures <- sshService.Serve(sshListener) }()
stop, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
select {
case <-stop.Done():
case err := <-failures:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
slog.Error("server stopped", "error", err)
}
}
shutdown, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second)
defer cancelShutdown()
_ = httpServer.Shutdown(shutdown)
_ = sshService.Close()
return nil
}
package main
import (
"fmt"
"net/url"
"regexp"
"strings"
)
var projectName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
// Repo is the project folder derived from a repository link or name.
type Repo struct {
Owner string
Name string
Project string
}
// ParseRepo accepts a full repository URL, owner/name, or a bare project
// name. A named owner must be the registered user.
func ParseRepo(input, user string) (Repo, error) {
input = strings.TrimSpace(input)
if input == "" {
return Repo{}, fmt.Errorf("enter a repository link or name")
}
var segments []string
if strings.Contains(input, "://") {
parsed, err := url.Parse(input)
if err != nil || parsed.Host == "" {
return Repo{}, fmt.Errorf("that is not a repository URL")
}
segments = splitURLPath(parsed.Path)
} else {
segments = splitURLPath(input)
}
var owner, name string
switch len(segments) {
case 1:
owner, name = user, segments[0]
case 2:
owner, name = segments[0], segments[1]
default:
return Repo{}, fmt.Errorf("use a repository URL or owner/name")
}
owner = strings.ToLower(owner)
if owner != user {
return Repo{}, fmt.Errorf("the repository owner must be %s", user)
}
name = strings.TrimSuffix(name, ".git")
if !projectName.MatchString(name) || name == "." || name == ".." || len(name) > 255 {
return Repo{}, fmt.Errorf("invalid repository name")
}
return Repo{Owner: owner, Name: name, Project: name}, nil
}
func splitURLPath(value string) []string {
value = strings.Trim(strings.TrimSpace(value), "/")
if value == "" {
return nil
}
return strings.Split(value, "/")
}
package main
import (
"bytes"
"context"
"embed"
"errors"
"fmt"
"html/template"
"io"
"mime"
"net/http"
"net/url"
"os"
"path"
"strings"
"syscall"
"time"
"uploadservice/internal/markdown"
)
//go:embed templates/*.html
var templateFS embed.FS
// Server is the HTTP front end: public tree and file reads, token-gated
// writes.
type Server struct {
cfg Config
store *Store
tokens *Token
github *GitHub
templates *template.Template
renderer *markdown.Renderer
mux *http.ServeMux
}
// NewServer parses the embedded templates and registers the routes.
func NewServer(cfg Config, store *Store, tokens *Token, github *GitHub) (*Server, error) {
templates, err := template.New("cdn").Funcs(template.FuncMap{
"mark": mark,
"human": humanSize,
"date": func(value time.Time) string {
if value.IsZero() {
return ""
}
return value.Local().Format("2006-01-02 15:04")
},
}).ParseFS(templateFS, "templates/*.html")
if err != nil {
return nil, err
}
server := &Server{
cfg: cfg,
store: store,
tokens: tokens,
github: github,
templates: templates,
renderer: markdown.NewRenderer(),
mux: http.NewServeMux(),
}
server.routes()
return server, nil
}
// Handler returns the mux wrapped with the baseline security headers.
func (s *Server) Handler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Referrer-Policy", "no-referrer")
s.mux.ServeHTTP(w, r)
})
}
func (s *Server) routes() {
s.mux.HandleFunc("GET /{$}", s.landing)
s.mux.HandleFunc("GET /healthz", s.health)
s.mux.HandleFunc("GET /favicon.svg", s.favicon)
s.mux.HandleFunc("GET /favicon.ico", s.favicon)
s.mux.HandleFunc("POST /register", s.register)
s.mux.HandleFunc("POST /upload", s.upload)
s.mux.HandleFunc("PUT /{user}/{path...}", s.put)
s.mux.HandleFunc("GET /{user}/{path...}", s.browse)
}
func (s *Server) landing(w http.ResponseWriter, r *http.Request) {
page := &Page{
Title: "cdn.si",
SSHHost: s.cfg.SSHHost,
SSHPort: s.cfg.SSHPort(),
PublicURL: s.cfg.PublicURL,
Trees: s.store.UserTrees(),
}
s.render(w, "landing.html", page, http.StatusOK)
}
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintln(w, "ok")
}
// favicon serves the mark itself. Browsers ask for /favicon.ico by default,
// so both names answer with the SVG.
func (s *Server) favicon(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "public, max-age=86400")
_, _ = io.WriteString(w, logoSVG)
}
func (s *Server) register(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
user, err := NormalizeUser(r.FormValue("user"))
if err != nil {
s.render(w, "landing.html", s.landingPage("Use a GitHub username (letters, digits and single hyphens)."), http.StatusBadRequest)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
defer cancel()
keys, err := s.github.Keys(ctx, user)
if err != nil {
s.render(w, "landing.html", s.landingPage("Could not read keys for "+user+": "+err.Error()), http.StatusBadGateway)
return
}
if err := s.store.SaveKeys(user, keys); err != nil {
s.render(w, "landing.html", s.landingPage("Could not store keys: "+err.Error()), http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/"+user+"/", http.StatusSeeOther)
}
func (s *Server) browse(w http.ResponseWriter, r *http.Request) {
user, err := NormalizeUser(r.PathValue("user"))
if err != nil {
http.NotFound(w, r)
return
}
segments, err := s.store.SplitPath(r.PathValue("path"))
if err != nil {
http.NotFound(w, r)
return
}
lock := s.store.Lock(user)
lock.Tree.RLock()
root, err := s.store.Open(user)
if err != nil {
lock.Tree.RUnlock()
http.NotFound(w, r)
return
}
defer root.Close()
target := strings.Join(segments, "/")
if target == "" {
target = "."
}
info, err := root.Stat(target)
if err != nil {
lock.Tree.RUnlock()
http.NotFound(w, r)
return
}
if info.IsDir() {
if !strings.HasSuffix(r.URL.Path, "/") {
lock.Tree.RUnlock()
location := r.URL.EscapedPath() + "/"
if r.URL.RawQuery != "" {
location += "?" + r.URL.RawQuery
}
http.Redirect(w, r, location, http.StatusMovedPermanently)
return
}
dir, err := root.Open(target)
if err != nil {
lock.Tree.RUnlock()
http.NotFound(w, r)
return
}
rawEntries, err := dir.ReadDir(-1)
dir.Close()
if err != nil {
lock.Tree.RUnlock()
http.Error(w, "cannot read directory", http.StatusInternalServerError)
return
}
usage, _, err := s.store.Usage(user)
if err != nil {
lock.Tree.RUnlock()
http.Error(w, "cannot read usage", http.StatusInternalServerError)
return
}
entries := make([]Entry, 0, len(rawEntries))
base := "/" + user + "/" + escapedJoin(segments, "/", true)
for _, raw := range rawEntries {
name := raw.Name()
if strings.HasPrefix(name, tempPrefix) {
continue
}
item, err := raw.Info()
if err != nil {
continue
}
href := base + url.PathEscape(name)
if raw.IsDir() {
href += "/"
}
entries = append(entries, Entry{
Name: name,
Href: href,
Dir: raw.IsDir(),
Link: item.Mode()&os.ModeSymlink != 0,
Size: item.Size(),
ModTime: item.ModTime(),
})
}
lock.Tree.RUnlock()
SortEntries(entries)
page := s.pageFor(user, segments, entries, usage)
if repoInput := r.URL.Query().Get("repo"); repoInput != "" {
page.Repo = repoInput
repo, err := ParseRepo(repoInput, user)
if err != nil {
page.Error = err.Error()
} else {
page.SyncCommand = s.cfg.RSyncCommand(user, repo.Project)
}
}
if uploaded := r.URL.Query().Get("uploaded"); uploaded != "" {
page.Notice = "Uploaded " + uploaded + " file(s)."
}
s.render(w, "user.html", page, http.StatusOK)
return
}
file, err := root.OpenFile(target, os.O_RDONLY|syscall.O_NONBLOCK, 0)
if err != nil {
lock.Tree.RUnlock()
http.NotFound(w, r)
return
}
stat, err := file.Stat()
if err != nil || !stat.Mode().IsRegular() {
file.Close()
lock.Tree.RUnlock()
http.NotFound(w, r)
return
}
lock.Tree.RUnlock()
defer file.Close()
extension := strings.ToLower(path.Ext(target))
if extension == ".md" {
s.document(w, r, user, segments, file)
return
}
if servesAsSource(extension) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
http.ServeContent(w, r, stat.Name(), stat.ModTime(), file)
return
}
if contentType := mime.TypeByExtension(extension); contentType != "" {
w.Header().Set("Content-Type", contentType)
}
http.ServeContent(w, r, stat.Name(), stat.ModTime(), file)
}
// sourceExtensions are served as plain text and never as a document. The
// content is readable, but a browser must not render or run it: a published
// page would execute on the site's origin. A markdown file is the way to
// publish a page, and it is rendered without raw HTML.
var sourceExtensions = map[string]bool{
".html": true, ".htm": true, ".xhtml": true,
}
// servesAsSource reports whether a file is served as the text it is.
func servesAsSource(extension string) bool {
return sourceExtensions[extension]
}
// markdownLimit is the largest file rendered as a page. A body past it is
// served as the text it is, so one upload cannot spend the server's memory on
// a single request.
const markdownLimit = 1 << 20
// document answers a markdown file as an HTML page. The file name is the
// address, so links written next to it resolve beside it.
func (s *Server) document(w http.ResponseWriter, r *http.Request, user string, segments []string, file *os.File) {
stat, err := file.Stat()
if err != nil {
http.Error(w, "cannot read file", http.StatusInternalServerError)
return
}
content, err := io.ReadAll(io.LimitReader(file, markdownLimit+1))
if err != nil {
http.Error(w, "cannot read file", http.StatusInternalServerError)
return
}
if len(content) > markdownLimit {
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
http.ServeContent(w, r, stat.Name(), stat.ModTime(), file)
return
}
page := s.documentPage(user, segments)
page.Content = template.HTML(s.renderer.Render(content))
s.render(w, "document.html", page, http.StatusOK)
}
// documentPage is the frame around a rendered document: the user, the path
// to the file and the file name as the title.
func (s *Server) documentPage(user string, segments []string) *Page {
crumbs := []Entry{{Name: user, Href: "/" + user + "/", Dir: true}}
current := make([]string, 0, len(segments))
for index, segment := range segments {
current = append(current, segment)
href := "/" + user + "/" + escapedJoin(current, "/", false)
if index < len(segments)-1 {
href += "/"
}
crumbs = append(crumbs, Entry{Name: segment, Href: href, Dir: index < len(segments)-1})
}
name := segments[len(segments)-1]
return &Page{
Title: name + " · " + user + " · cdn.si",
User: user,
Breadcrumbs: crumbs,
Path: strings.Join(segments, "/"),
}
}
func (s *Server) upload(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.Quota*2+(1<<20))
if err := r.ParseMultipartForm(32 << 20); err != nil {
s.writeError(w, r, http.StatusBadRequest, "Could not read the upload: "+err.Error())
return
}
defer r.MultipartForm.RemoveAll()
token := r.FormValue("token")
if token == "" {
token = bearerToken(r)
}
user, err := NormalizeUser(r.FormValue("user"))
if err != nil {
s.writeError(w, r, http.StatusBadRequest, "Invalid username.")
return
}
tokenUser, err := s.tokens.Verify(token)
if err != nil || tokenUser != user {
s.writeError(w, r, http.StatusUnauthorized, "The upload token is missing, invalid or expired. Get one with: "+s.cfg.SSHCommand(user)+" token")
return
}
dir, err := s.store.SplitPath(r.FormValue("path"))
if err != nil {
s.writeError(w, r, http.StatusBadRequest, "Invalid upload path.")
return
}
files := r.MultipartForm.File["file"]
if len(files) == 0 {
s.writeError(w, r, http.StatusBadRequest, "No files were sent.")
return
}
if len(files) > s.cfg.MaxUploadFiles {
s.writeError(w, r, http.StatusRequestEntityTooLarge, "Too many files in one upload.")
return
}
uploads := make([]Upload, 0, len(files))
for _, header := range files {
file, err := header.Open()
if err != nil {
s.writeError(w, r, http.StatusBadRequest, "Could not read "+header.Filename+".")
return
}
defer file.Close()
uploads = append(uploads, Upload{Name: header.Filename, Body: file})
}
count, err := s.store.WriteBatch(user, dir, uploads)
if err != nil {
s.writeStoreError(w, r, err)
return
}
location := "/" + user + "/" + escapedJoin(dir, "/", true) + "?uploaded=" + fmt.Sprint(count)
http.Redirect(w, r, location, http.StatusSeeOther)
}
func (s *Server) put(w http.ResponseWriter, r *http.Request) {
user, err := NormalizeUser(r.PathValue("user"))
if err != nil {
http.NotFound(w, r)
return
}
rawPath := r.PathValue("path")
if strings.HasSuffix(rawPath, "/") {
http.Error(w, "PUT needs a file path", http.StatusBadRequest)
return
}
segments, err := s.store.SplitPath(rawPath)
if err != nil || len(segments) == 0 {
http.Error(w, "invalid path", http.StatusBadRequest)
return
}
tokenUser, err := s.tokens.Verify(bearerToken(r))
if err != nil || tokenUser != user {
http.Error(w, "missing or invalid token", http.StatusUnauthorized)
return
}
if r.ContentLength > s.cfg.Quota {
http.Error(w, "quota exceeded", http.StatusRequestEntityTooLarge)
return
}
body := http.MaxBytesReader(w, r.Body, s.cfg.Quota+1)
upload := Upload{Name: segments[len(segments)-1], Body: body}
_, err = s.store.WriteBatch(user, segments[:len(segments)-1], []Upload{upload})
if err != nil {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
http.Error(w, "quota exceeded", http.StatusRequestEntityTooLarge)
return
}
switch {
case errors.Is(err, errQuota), errors.Is(err, errTooMany):
http.Error(w, "quota exceeded", http.StatusRequestEntityTooLarge)
case errors.Is(err, errInvalidPath), errors.Is(err, errInvalidName), errors.Is(err, errDuplicate):
http.Error(w, err.Error(), http.StatusBadRequest)
case errors.Is(err, errConflict):
http.Error(w, err.Error(), http.StatusConflict)
default:
http.Error(w, "write failed", http.StatusInternalServerError)
}
return
}
w.WriteHeader(http.StatusCreated)
fmt.Fprintln(w, "stored "+strings.Join(segments, "/"))
}
func (s *Server) pageFor(user string, segments []string, entries []Entry, usage int64) *Page {
pathSlash := escapedJoin(segments, "/", true)
base := "/" + user + "/" + pathSlash
crumbs := []Entry{{Name: user, Href: "/" + user + "/", Dir: true}}
current := make([]string, 0, len(segments))
for _, segment := range segments {
current = append(current, segment)
crumbs = append(crumbs, Entry{
Name: segment,
Href: "/" + user + "/" + escapedJoin(current, "/", true),
Dir: true,
})
}
parent := "/"
if len(segments) > 0 {
parent = "/" + user + "/" + escapedJoin(segments[:len(segments)-1], "/", true)
}
percent := 0
if s.cfg.Quota > 0 {
percent = int(usage * 100 / s.cfg.Quota)
if percent > 100 {
percent = 100
}
}
return &Page{
Title: user + " · cdn.si",
User: user,
Keys: s.store.KeyFingerprints(user),
Usage: usage,
Quota: s.cfg.Quota,
Percent: percent,
Path: strings.Join(segments, "/"),
PathSlash: pathSlash,
BaseHref: base,
Breadcrumbs: crumbs,
Parent: parent,
Entries: entries,
SSHHost: s.cfg.SSHHost,
SSHPort: s.cfg.SSHPort(),
SSHCommand: s.cfg.SSHCommand(user),
TokenCommand: s.cfg.SSHCommand(user) + " token",
WhoamiCommand: s.cfg.SSHCommand(user) + " whoami",
CurlCommand: fmt.Sprintf("curl -T file -H \"Authorization: Bearer $TOKEN\" %s/%s/%sfile", s.cfg.PublicURL, user, pathSlash),
}
}
func (s *Server) writeStoreError(w http.ResponseWriter, r *http.Request, err error) {
switch {
case errors.Is(err, errQuota), errors.Is(err, errTooMany):
s.writeError(w, r, http.StatusRequestEntityTooLarge, "The 50 MB quota would be exceeded.")
case errors.Is(err, errInvalidPath), errors.Is(err, errInvalidName), errors.Is(err, errDuplicate):
s.writeError(w, r, http.StatusBadRequest, err.Error())
case errors.Is(err, errConflict):
s.writeError(w, r, http.StatusConflict, "A directory already exists at that path.")
default:
s.writeError(w, r, http.StatusInternalServerError, "Could not store the upload.")
}
}
func (s *Server) writeError(w http.ResponseWriter, r *http.Request, status int, message string) {
user, _ := NormalizeUser(r.FormValue("user"))
page := s.landingPage(message)
page.User = user
s.render(w, "landing.html", page, status)
}
func (s *Server) landingPage(message string) *Page {
return &Page{
Title: "cdn.si",
Error: message,
SSHHost: s.cfg.SSHHost,
SSHPort: s.cfg.SSHPort(),
PublicURL: s.cfg.PublicURL,
Trees: s.store.UserTrees(),
}
}
func (s *Server) render(w http.ResponseWriter, name string, page *Page, status int) {
var buffer bytes.Buffer
if err := s.templates.ExecuteTemplate(&buffer, name, page); err != nil {
http.Error(w, "template error", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Frame-Options", "DENY")
w.WriteHeader(status)
_, _ = w.Write(buffer.Bytes())
}
func bearerToken(r *http.Request) string {
value := r.Header.Get("Authorization")
const prefix = "Bearer "
if len(value) > len(prefix) && strings.EqualFold(value[:len(prefix)], prefix) {
return strings.TrimSpace(value[len(prefix):])
}
return ""
}
func escapedJoin(segments []string, separator string, trailing bool) string {
parts := make([]string, 0, len(segments))
for _, segment := range segments {
parts = append(parts, url.PathEscape(segment))
}
joined := strings.Join(parts, separator)
if trailing && joined != "" {
joined += separator
}
return joined
}
func humanSize(size int64) string {
if size < 1024 {
return fmt.Sprintf("%d B", size)
}
divider := int64(1024)
exponent := 0
for size/divider >= 1024 && exponent < 4 {
divider *= 1024
exponent++
}
return fmt.Sprintf("%.1f %ciB", float64(size)/float64(divider), "KMGTP"[exponent])
}
package main
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/pem"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/gliderlabs/ssh"
xssh "golang.org/x/crypto/ssh"
)
// SSHD is the in-process SSH server. It authenticates the connecting username
// against the keys that user publishes on GitHub, caching the document in the
// data directory, and offers exactly three things: an upload token, a whoami
// probe, and rsync.
type SSHD struct {
cfg Config
store *Store
tokens *Token
keys *KeyCache
server *ssh.Server
}
// NewSSHD loads or creates the host key and configures the server.
func NewSSHD(cfg Config, store *Store, tokens *Token, github *GitHub) (*SSHD, error) {
signer, err := loadHostKey(store.sshDir)
if err != nil {
return nil, err
}
service := &SSHD{
cfg: cfg,
store: store,
tokens: tokens,
keys: NewKeyCache(github, store, cfg.KeysTTL, cfg.KeysInterval),
}
service.server = &ssh.Server{
Handler: service.handle,
PublicKeyHandler: service.auth,
HostSigners: []ssh.Signer{signer},
Version: "uploadservice",
IdleTimeout: 10 * time.Minute,
MaxTimeout: 8 * time.Hour,
PtyCallback: func(ssh.Context, ssh.Pty) bool { return false },
SessionRequestCallback: func(_ ssh.Session, requestType string) bool {
return requestType == "shell" || requestType == "exec"
},
}
return service, nil
}
// Serve accepts connections on l until the server is closed.
func (s *SSHD) Serve(l net.Listener) error {
return s.server.Serve(l)
}
// Close stops the listener and all sessions.
func (s *SSHD) Close() error {
return s.server.Close()
}
func (s *SSHD) auth(ctx ssh.Context, key ssh.PublicKey) bool {
user, err := NormalizeUser(ctx.User())
if err != nil {
return false
}
return s.keys.Verify(ctx, user, key)
}
func (s *SSHD) handle(session ssh.Session) {
user, err := NormalizeUser(session.User())
if err != nil {
fmt.Fprintln(session.Stderr(), "unknown user")
_ = session.Exit(1)
return
}
if _, _, ok := session.Pty(); ok {
fmt.Fprintln(session.Stderr(), "interactive sessions are not supported")
_ = session.Exit(1)
return
}
raw := strings.TrimSpace(session.RawCommand())
switch {
case raw == "token":
token, expiry := s.tokens.Issue(user)
fmt.Fprintln(session, token)
fmt.Fprintf(session.Stderr(), "token expires %s\n", expiry.UTC().Format(time.RFC3339))
_ = session.Exit(0)
case raw == "whoami":
fingerprint := ""
if key := session.PublicKey(); key != nil {
fingerprint = xssh.FingerprintSHA256(key)
}
usage, entries, err := s.store.Usage(user)
if err != nil {
fmt.Fprintf(session.Stderr(), "usage: %v\n", err)
_ = session.Exit(1)
return
}
fmt.Fprintf(session, "%s %s\n", user, fingerprint)
fmt.Fprintf(session, "%s of %s used, %d entries\n", humanSize(usage), humanSize(s.store.quota), entries)
_ = session.Exit(0)
case raw == "rsync --server" || strings.HasPrefix(raw, "rsync --server "):
s.runRsync(session, user, raw)
default:
fmt.Fprintln(session.Stderr(), "supported commands: token, whoami, rsync")
_ = session.Exit(1)
}
}
func (s *SSHD) runRsync(session ssh.Session, user, raw string) {
lock := s.store.Lock(user)
lock.Write.Lock()
defer lock.Write.Unlock()
usage, _, err := s.store.Usage(user)
if err != nil {
fmt.Fprintf(session.Stderr(), "usage: %v\n", err)
_ = session.Exit(1)
return
}
if usage >= s.store.quota {
fmt.Fprintln(session.Stderr(), "quota exceeded: delete files before publishing")
_ = session.Exit(2)
return
}
stage, err := s.store.NewStage(user)
if err != nil {
fmt.Fprintf(session.Stderr(), "stage: %v\n", err)
_ = session.Exit(1)
return
}
defer stage.Discard()
ctx, cancel := context.WithCancel(session.Context())
defer cancel()
cmd := exec.CommandContext(ctx, s.cfg.RRsyncPath, "-wo", stage.Path())
cmd.Dir = "/"
cmd.Env = []string{
"PATH=/usr/local/bin:/usr/bin:/bin",
"HOME=" + s.store.dataDir,
"TMPDIR=" + s.store.tmpDir,
"LANG=C",
"SSH_ORIGINAL_COMMAND=" + raw,
"SSH_CONNECTION=" + session.RemoteAddr().String(),
}
stdin, err := cmd.StdinPipe()
if err != nil {
fmt.Fprintf(session.Stderr(), "rsync: %v\n", err)
_ = session.Exit(1)
return
}
cmd.Stdout = session
cmd.Stderr = session.Stderr()
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Cancel = func() error {
return killGroup(cmd)
}
cmd.WaitDelay = 5 * time.Second
if err := cmd.Start(); err != nil {
fmt.Fprintf(session.Stderr(), "rsync: %v\n", err)
_ = session.Exit(1)
return
}
go func() {
_, _ = io.Copy(stdin, session)
_ = stdin.Close()
}()
done := make(chan struct{})
monitorDone := make(chan struct{})
var killed atomic.Bool
limit := s.store.quota*2 + 64<<20
go func() {
defer close(monitorDone)
ticker := time.NewTicker(s.cfg.MonitorEvery)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ticker.C:
size, entries, err := stage.Usage()
if err != nil && !errors.Is(err, errTooMany) && !errors.Is(err, errTooDeep) {
continue
}
if size > limit || entries > s.store.maxEntries || errors.Is(err, errTooMany) || errors.Is(err, errTooDeep) {
killed.Store(true)
_ = killGroup(cmd)
cancel()
return
}
}
}
}()
waitErr := cmd.Wait()
close(done)
<-monitorDone
if killed.Load() {
fmt.Fprintln(session.Stderr(), "quota exceeded: staging passed the safety limit")
_ = session.Exit(2)
return
}
if waitErr != nil {
var exitErr *exec.ExitError
if errors.As(waitErr, &exitErr) {
fmt.Fprintf(session.Stderr(), "rsync exited: %v\n", exitErr)
_ = session.Exit(exitCode(exitErr))
return
}
fmt.Fprintf(session.Stderr(), "rsync: %v\n", waitErr)
_ = session.Exit(1)
return
}
if err := stage.Validate(); err != nil {
fmt.Fprintf(session.Stderr(), "publish rejected: %v\n", err)
if errors.Is(err, errQuota) {
_ = session.Exit(2)
return
}
_ = session.Exit(1)
return
}
if err := stage.Commit(); err != nil {
fmt.Fprintf(session.Stderr(), "commit: %v\n", err)
_ = session.Exit(1)
return
}
_ = session.Exit(0)
}
func killGroup(cmd *exec.Cmd) error {
if cmd.Process == nil {
return nil
}
return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
}
func exitCode(err *exec.ExitError) int {
code := err.ExitCode()
if code <= 0 || code > 125 {
return 23
}
return code
}
func loadHostKey(dir string) (xssh.Signer, error) {
name := filepath.Join(dir, "host_ed25519")
if data, err := os.ReadFile(name); err == nil {
return xssh.ParsePrivateKey(data)
}
_, private, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return nil, err
}
block, err := xssh.MarshalPrivateKey(private, "")
if err != nil {
return nil, err
}
data := pem.EncodeToMemory(block)
if err := os.WriteFile(name, data, 0o600); err != nil {
return nil, err
}
return xssh.NewSignerFromKey(private)
}
package main
import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/sys/unix"
)
// Stage is a private copy of one user's tree that an rsync session writes
// into. Nothing reaches the live tree until the whole session fits the
// quota and passes validation.
type Stage struct {
store *Store
user string
path string
committed bool
}
// NewStage clones the live tree into /data/_staging.
func (s *Store) NewStage(user string) (*Stage, error) {
user, err := NormalizeUser(user)
if err != nil {
return nil, err
}
stagePath := filepath.Join(s.stagingDir, user+"-"+randomHex(8))
if err := os.Mkdir(stagePath, 0o700); err != nil {
return nil, err
}
if err := copyTree(s.UserDir(user), stagePath); err != nil {
os.RemoveAll(stagePath)
return nil, err
}
return &Stage{store: s, user: user, path: stagePath}, nil
}
// Path is the staging directory handed to rrsync.
func (st *Stage) Path() string { return st.path }
// Usage returns the staged apparent size and entry count.
func (st *Stage) Usage() (int64, int, error) {
return st.store.treeStats(st.path)
}
// Validate enforces the quota, entry/depth limits and symlink rules at
// commit time.
func (st *Stage) Validate() error {
size, _, err := st.store.treeStats(st.path)
if err != nil {
return err
}
if size > st.store.quota {
return errQuota
}
return filepath.WalkDir(st.path, func(name string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if name == st.path {
return nil
}
info, err := entry.Info()
if err != nil {
return err
}
switch {
case entry.IsDir(), info.Mode().IsRegular():
return nil
case info.Mode()&fs.ModeSymlink != 0:
target, err := os.Readlink(name)
if err != nil {
return err
}
if filepath.IsAbs(target) {
return errSymlinkEscape
}
resolved := filepath.Join(filepath.Dir(name), target)
rel, err := filepath.Rel(st.path, resolved)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return errSymlinkEscape
}
return nil
default:
return errSpecialFile
}
})
}
// Commit swaps the staged tree into place. RENAME_EXCHANGE makes the swap
// atomic; the two-rename fallback runs under the tree lock, so readers never
// observe the gap.
func (st *Stage) Commit() error {
live := st.store.UserDir(st.user)
lock := st.store.Lock(st.user)
lock.Tree.Lock()
defer lock.Tree.Unlock()
info, err := os.Lstat(live)
switch {
case errors.Is(err, fs.ErrNotExist):
if err := os.MkdirAll(live, 0o755); err != nil {
return err
}
case err != nil:
return err
case !info.IsDir():
return errConflict
}
err = unix.Renameat2(unix.AT_FDCWD, st.path, unix.AT_FDCWD, live, unix.RENAME_EXCHANGE)
if err == nil {
st.committed = true
_ = os.Chmod(live, 0o755)
_ = os.RemoveAll(st.path)
return nil
}
trash := filepath.Join(st.store.trashDir, st.user+"-"+randomHex(8))
if err := os.Rename(live, trash); err != nil {
return err
}
if err := os.Rename(st.path, live); err != nil {
_ = os.Rename(trash, live)
return err
}
st.committed = true
_ = os.Chmod(live, 0o755)
_ = os.RemoveAll(trash)
return nil
}
// Discard removes the staging directory unless it was committed.
func (st *Stage) Discard() {
if st.committed {
return
}
_ = os.RemoveAll(st.path)
}
func copyTree(src, dst string) error {
info, err := os.Lstat(src)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return err
}
if !info.IsDir() {
return errConflict
}
type directory struct {
path string
modTime time.Time
}
var directories []directory
err = filepath.WalkDir(src, func(name string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(src, name)
if err != nil {
return err
}
if rel == "." {
return nil
}
target := filepath.Join(dst, rel)
info, err := entry.Info()
if err != nil {
return err
}
switch {
case entry.IsDir():
if err := os.Mkdir(target, info.Mode().Perm()|0o700); err != nil {
return err
}
directories = append(directories, directory{target, info.ModTime()})
return nil
case info.Mode()&fs.ModeSymlink != 0:
link, err := os.Readlink(name)
if err != nil {
return err
}
return os.Symlink(link, target)
case info.Mode().IsRegular():
return copyFile(name, target, info)
default:
return errSpecialFile
}
})
if err != nil {
return err
}
for i := len(directories) - 1; i >= 0; i-- {
_ = os.Chtimes(directories[i].path, directories[i].modTime, directories[i].modTime)
}
return nil
}
func copyFile(src, dst string, info fs.FileInfo) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm())
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
return os.Chtimes(dst, info.ModTime(), info.ModTime())
}
package main
import (
"bytes"
"crypto/rand"
"crypto/subtle"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
"unicode"
"golang.org/x/crypto/ssh"
)
const tempPrefix = ".cdn-tmp-"
var usernamePattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?$`)
// Store owns the data directory: user trees, cached public keys, the host
// key, staging and temporary space.
type Store struct {
dataDir string
keysDir string
sshDir string
stagingDir string
tmpDir string
trashDir string
quota int64
maxEntries int
maxDepth int
mu sync.Mutex
locks map[string]*UserLock
}
// NewStore returns a store rooted at cfg.DataDir.
func NewStore(cfg Config) *Store {
root := cfg.DataDir
return &Store{
dataDir: root,
keysDir: filepath.Join(root, "_keys"),
sshDir: filepath.Join(root, "_ssh"),
stagingDir: filepath.Join(root, "_staging"),
tmpDir: filepath.Join(root, "_tmp"),
trashDir: filepath.Join(root, "_trash"),
quota: cfg.Quota,
maxEntries: cfg.MaxEntries,
maxDepth: cfg.MaxDepth,
locks: make(map[string]*UserLock),
}
}
// Init creates the directory skeleton and clears transient state left by a
// previous process.
func (s *Store) Init() error {
for _, dir := range []string{s.dataDir, s.keysDir, s.sshDir, s.stagingDir, s.tmpDir, s.trashDir} {
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
}
if err := os.Chmod(s.sshDir, 0o700); err != nil {
return err
}
for _, dir := range []string{s.stagingDir, s.trashDir} {
if err := clearDir(dir); err != nil {
return err
}
}
if err := clearDir(s.tmpDir); err != nil {
return err
}
return s.sweepUserTemps()
}
// NormalizeUser lowercases and validates a GitHub username.
func NormalizeUser(raw string) (string, error) {
user := strings.ToLower(strings.TrimSpace(raw))
if len(user) == 0 || len(user) > 39 || !usernamePattern.MatchString(user) || strings.Contains(user, "--") {
return "", errInvalidUser
}
return user, nil
}
// SplitPath validates and splits a slash-separated path below a user root.
func (s *Store) SplitPath(raw string) ([]string, error) {
raw = strings.Trim(raw, "/")
if raw == "" {
return nil, nil
}
segments := strings.Split(raw, "/")
if len(segments) > s.maxDepth {
return nil, errTooDeep
}
for _, segment := range segments {
if err := validateSegment(segment); err != nil {
return nil, err
}
}
return segments, nil
}
func validateSegment(segment string) error {
if segment == "" || segment == "." || segment == ".." || len(segment) > 255 {
return errInvalidPath
}
for _, r := range segment {
if r == 0 || r == '/' || r == '\\' || unicode.IsControl(r) {
return errInvalidPath
}
}
if strings.HasPrefix(segment, tempPrefix) {
return errInvalidPath
}
return nil
}
// UserDir is the live tree for a normalised username.
func (s *Store) UserDir(user string) string {
return filepath.Join(s.dataDir, user)
}
// UserTrees lists the published trees, one per user. The store's own folders
// start with an underscore and are not trees; a folder that is not a valid
// user name is not one either.
func (s *Store) UserTrees() []Entry {
rawEntries, err := os.ReadDir(s.dataDir)
if err != nil {
return nil
}
trees := make([]Entry, 0, len(rawEntries))
for _, raw := range rawEntries {
if !raw.IsDir() {
continue
}
user, err := NormalizeUser(raw.Name())
if err != nil || user != raw.Name() {
continue
}
info, err := raw.Info()
if err != nil {
continue
}
trees = append(trees, Entry{
Name: user,
Href: "/" + url.PathEscape(user) + "/",
Dir: true,
ModTime: info.ModTime(),
})
}
SortEntries(trees)
return trees
}
// KeysPath is /data/_keys/<user>.pub.
func (s *Store) KeysPath(user string) string {
return filepath.Join(s.keysDir, user+".pub")
}
// Lock returns the per-user locks.
func (s *Store) Lock(user string) *UserLock {
s.mu.Lock()
defer s.mu.Unlock()
lock, ok := s.locks[user]
if !ok {
lock = &UserLock{}
s.locks[user] = lock
}
return lock
}
// SaveKeys atomically stores the raw public key document for user.
func (s *Store) SaveKeys(user string, data []byte) error {
user, err := NormalizeUser(user)
if err != nil {
return err
}
if err := os.MkdirAll(s.keysDir, 0o755); err != nil {
return err
}
if err := os.MkdirAll(s.UserDir(user), 0o755); err != nil {
return err
}
tmp, err := os.CreateTemp(s.keysDir, tempPrefix+"keys-*")
if err != nil {
return err
}
tmpName := tmp.Name()
defer os.Remove(tmpName)
if _, err := tmp.Write(data); err != nil {
tmp.Close()
return err
}
if err := tmp.Chmod(0o644); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmpName, s.KeysPath(user))
}
// Keys parses the cached public keys of user.
func (s *Store) Keys(user string) ([]ssh.PublicKey, error) {
data, err := os.ReadFile(s.KeysPath(user))
if err != nil {
return nil, err
}
return ParseAuthorizedKeys(data)
}
// KeyFingerprints returns SHA256 fingerprints for display.
func (s *Store) KeyFingerprints(user string) []string {
keys, err := s.Keys(user)
if err != nil {
return nil
}
fingerprints := make([]string, 0, len(keys))
for _, key := range keys {
fingerprints = append(fingerprints, ssh.FingerprintSHA256(key))
}
return fingerprints
}
// KeyEqual reports whether presented matches any key stored for user.
func (s *Store) KeyEqual(user string, presented ssh.PublicKey) bool {
keys, err := s.Keys(user)
if err != nil {
return false
}
want := presented.Marshal()
for _, key := range keys {
if subtle.ConstantTimeCompare(key.Marshal(), want) == 1 {
return true
}
}
return false
}
// ParseAuthorizedKeys parses every usable line of an authorized_keys file.
func ParseAuthorizedKeys(data []byte) ([]ssh.PublicKey, error) {
var keys []ssh.PublicKey
for _, line := range bytes.Split(data, []byte("\n")) {
line = bytes.TrimSpace(line)
if len(line) == 0 || line[0] == '#' {
continue
}
key, _, _, _, err := ssh.ParseAuthorizedKey(line)
if err != nil {
continue
}
keys = append(keys, key)
}
return keys, nil
}
// Open returns a confinement root for the user's live tree.
func (s *Store) Open(user string) (*os.Root, error) {
dir := s.UserDir(user)
info, err := os.Lstat(dir)
if err != nil {
return nil, err
}
if !info.IsDir() {
return nil, errNotFound
}
return os.OpenRoot(dir)
}
// Usage returns the apparent size and entry count of the live tree.
func (s *Store) Usage(user string) (int64, int, error) {
return s.treeStats(s.UserDir(user))
}
func (s *Store) treeStats(root string) (int64, int, error) {
info, err := os.Lstat(root)
if errors.Is(err, fs.ErrNotExist) {
return 0, 0, nil
}
if err != nil {
return 0, 0, err
}
if !info.IsDir() {
return 0, 0, errNotFound
}
var size int64
entries := 0
err = filepath.WalkDir(root, func(name string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if name == root {
return nil
}
entries++
if entries > s.maxEntries {
return errTooMany
}
rel, relErr := filepath.Rel(root, name)
if relErr != nil {
return relErr
}
if depth := len(strings.Split(rel, string(filepath.Separator))); depth > s.maxDepth {
return errTooDeep
}
info, err := entry.Info()
if err != nil {
return err
}
if info.Mode().IsRegular() {
size += info.Size()
}
return nil
})
return size, entries, err
}
// WriteBatch stores a set of files below dir with an all-or-nothing quota
// check. Temporary files live inside the target directory so the final
// rename stays confined by os.Root.
func (s *Store) WriteBatch(user string, dir []string, uploads []Upload) (int, error) {
user, err := NormalizeUser(user)
if err != nil {
return 0, err
}
if len(uploads) == 0 {
return 0, errInvalidPath
}
for _, segment := range dir {
if err := validateSegment(segment); err != nil {
return 0, err
}
}
lock := s.Lock(user)
lock.Write.Lock()
defer lock.Write.Unlock()
if err := os.MkdirAll(s.UserDir(user), 0o755); err != nil {
return 0, err
}
root, err := s.Open(user)
if err != nil {
return 0, err
}
defer root.Close()
usage, entries, err := s.Usage(user)
if err != nil {
return 0, err
}
type target struct {
rel string
temp string
size int64
}
var targets []target
seen := make(map[string]struct{}, len(uploads))
var replaced, newTotal int64
newFiles := 0
cleanup := func() {
for _, item := range targets {
_ = root.Remove(item.temp)
}
}
committed := false
defer func() {
if !committed {
cleanup()
}
}()
for _, upload := range uploads {
name, err := cleanUploadName(upload.Name)
if err != nil {
return 0, err
}
segments := append(append([]string{}, dir...), name)
if len(segments) > s.maxDepth {
return 0, errTooDeep
}
rel := strings.Join(segments, "/")
if _, ok := seen[rel]; ok {
return 0, errDuplicate
}
seen[rel] = struct{}{}
info, err := root.Lstat(rel)
switch {
case err == nil:
if !info.Mode().IsRegular() {
return 0, errConflict
}
replaced += info.Size()
case errors.Is(err, fs.ErrNotExist):
newFiles++
default:
return 0, err
}
parent := path.Dir(rel)
if parent != "." {
if err := root.MkdirAll(parent, 0o755); err != nil {
return 0, err
}
}
temp := path.Join(parent, tempPrefix+randomHex(8))
if parent == "." {
temp = tempPrefix + randomHex(8)
}
file, err := root.OpenFile(temp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return 0, err
}
size, copyErr := io.Copy(file, upload.Body)
closeErr := file.Close()
if copyErr != nil {
_ = root.Remove(temp)
return 0, copyErr
}
if closeErr != nil {
_ = root.Remove(temp)
return 0, closeErr
}
targets = append(targets, target{rel: rel, temp: temp, size: size})
newTotal += size
}
projected := usage - replaced + newTotal
if projected > s.quota {
return 0, errQuota
}
if entries+newFiles > s.maxEntries {
return 0, errTooMany
}
for _, item := range targets {
if err := root.Rename(item.temp, item.rel); err != nil {
return 0, err
}
if err := root.Chmod(item.rel, 0o644); err != nil {
return 0, err
}
}
committed = true
return len(targets), nil
}
func cleanUploadName(name string) (string, error) {
name = strings.ReplaceAll(name, "\\", "/")
name = path.Base(name)
if name == "" || name == "." || name == ".." || strings.HasPrefix(name, tempPrefix) {
return "", errInvalidName
}
if err := validateSegment(name); err != nil {
return "", errInvalidName
}
return name, nil
}
// EnsureSecret returns the HMAC secret from env or /data/_ssh/token_secret,
// creating and persisting one when neither exists.
func (s *Store) EnsureSecret(env string) ([]byte, error) {
if env != "" {
return []byte(env), nil
}
name := filepath.Join(s.sshDir, "token_secret")
if data, err := os.ReadFile(name); err == nil && len(bytes.TrimSpace(data)) >= 16 {
return bytes.TrimSpace(data), nil
}
buf := make([]byte, 32)
if _, err := rand.Read(buf); err != nil {
return nil, err
}
secret := []byte(hex.EncodeToString(buf))
if err := os.WriteFile(name, secret, 0o600); err != nil {
return nil, err
}
return secret, nil
}
// TempDir is the multipart spill directory.
func (s *Store) TempDir() string { return s.tmpDir }
func (s *Store) sweepUserTemps() error {
entries, err := os.ReadDir(s.dataDir)
if err != nil {
return err
}
for _, entry := range entries {
name := entry.Name()
if strings.HasPrefix(name, "_") || !entry.IsDir() {
continue
}
root := filepath.Join(s.dataDir, name)
_ = filepath.WalkDir(root, func(path string, item fs.DirEntry, err error) error {
if err != nil {
return nil
}
if !item.IsDir() && strings.HasPrefix(item.Name(), tempPrefix) {
_ = os.Remove(path)
}
return nil
})
}
return nil
}
func clearDir(dir string) error {
entries, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, entry := range entries {
if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil {
return err
}
}
return nil
}
func randomHex(bytesCount int) string {
buf := make([]byte, bytesCount)
if _, err := rand.Read(buf); err != nil {
return fmt.Sprintf("%d", os.Getpid())
}
return hex.EncodeToString(buf)
}
package main
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"fmt"
"strconv"
"strings"
"time"
)
// Token issues and verifies the short-lived HMAC bearer tokens that authorise
// HTTP writes. No server-side state is kept; the SSH key login is what proves
// identity, the token only carries that proof to HTTP.
type Token struct {
secret []byte
ttl time.Duration
now func() time.Time
}
// NewToken returns a signer for the given secret.
func NewToken(secret []byte, ttl time.Duration) *Token {
return &Token{secret: secret, ttl: ttl, now: time.Now}
}
// Issue returns a token for user and its expiry time.
func (t *Token) Issue(user string) (string, time.Time) {
expiry := t.now().Add(t.ttl)
nonce := make([]byte, 12)
_, _ = rand.Read(nonce)
payload := fmt.Sprintf("v1|%s|%d|%s", user, expiry.Unix(), hex.EncodeToString(nonce))
return payload + "." + t.sign(payload), expiry
}
// Verify returns the user named by a valid, unexpired token.
func (t *Token) Verify(value string) (string, error) {
payload, signature, ok := strings.Cut(value, ".")
if !ok || payload == "" || signature == "" {
return "", errToken
}
if subtle.ConstantTimeCompare([]byte(signature), []byte(t.sign(payload))) != 1 {
return "", errToken
}
parts := strings.Split(payload, "|")
if len(parts) != 4 || parts[0] != "v1" {
return "", errToken
}
expiry, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil || t.now().Unix() > expiry {
return "", errToken
}
user, err := NormalizeUser(parts[1])
if err != nil || user != parts[1] {
return "", errToken
}
return user, nil
}
func (t *Token) sign(payload string) string {
mac := hmac.New(sha256.New, t.secret)
mac.Write([]byte(payload))
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
}