AJ ONeal
4 роки тому
9 змінених файлів з 514 додано та 0 видалено
@ -0,0 +1,3 @@ |
|||
*_vfsdata.go |
|||
.*.sw* |
|||
.DS_Store |
@ -0,0 +1 @@ |
|||
AJ ONeal <aj@therootcompany.com> (https://therootcompany.com) |
@ -0,0 +1,22 @@ |
|||
The MIT License (MIT) |
|||
|
|||
Copyright (c) 2020 The vfscopy Authors |
|||
Copyright (c) 2018 otiai10 |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
of this software and associated documentation files (the "Software"), to deal |
|||
in the Software without restriction, including without limitation the rights |
|||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
copies of the Software, and to permit persons to whom the Software is |
|||
furnished to do so, subject to the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be included in |
|||
all copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
|||
THE SOFTWARE. |
@ -0,0 +1,63 @@ |
|||
# [vfscopy](https://git.rootprojects.org/root/vfscopy) |
|||
|
|||
Recursively copy a Virtual FileSystem, such as |
|||
[http.FileSystem](https://golang.org/pkg/net/http/#FileSystem), |
|||
to a native file system destination. |
|||
|
|||
Works with any file system that implements http.FileSystem, |
|||
including `vfsgen`, `fileb0x`, `gobindata` and most others. |
|||
|
|||
## Example: native file system (os) |
|||
|
|||
```go |
|||
httpfs := http.Dir("/tmp/public/") |
|||
vfs := vfscopy.NewVFS(httpfs) |
|||
|
|||
if err := vfscopy.CopyAll(vfs, ".", "/tmp/dst/"); nil != err { |
|||
fmt.Fprintf(os.Stderr, "couldn't copy vfs: %v\n", err) |
|||
} |
|||
``` |
|||
|
|||
## Example: vfsgen |
|||
|
|||
**Note**: `vfsgen` does not support symlinks or file permissions. |
|||
|
|||
```go |
|||
package main |
|||
|
|||
import ( |
|||
"fmt" |
|||
|
|||
"git.rootprojects.org/root/vfscopy" |
|||
|
|||
// vfsgen-generated file system |
|||
"git.example.com/org/project/assets" |
|||
) |
|||
|
|||
func main() { |
|||
vfs := vfscopy.NewVFS(assets.Assets) |
|||
|
|||
if err := vfscopy.CopyAll(vfs, ".", "/tmp/dst/"); nil != err { |
|||
fmt.Fprintf(os.Stderr, "couldn't copy vfs: %v\n", err) |
|||
} |
|||
fmt.Println("Done.") |
|||
} |
|||
``` |
|||
|
|||
## Test |
|||
|
|||
```bash |
|||
# Generate the test virtual file system |
|||
go generate ./... |
|||
|
|||
# Run the tests |
|||
go test ./... |
|||
``` |
|||
|
|||
# License |
|||
|
|||
The MIT License (MIT) |
|||
|
|||
We used the recursive native file system copy implementation at |
|||
https://github.com/otiai10/copy as a starting point and added |
|||
virtual file system support. |
@ -0,0 +1,209 @@ |
|||
package vfscopy |
|||
|
|||
import ( |
|||
"io" |
|||
"os" |
|||
"path/filepath" |
|||
) |
|||
|
|||
const ( |
|||
// tmpPermissionForDirectory makes the destination directory writable,
|
|||
// so that stuff can be copied recursively even if any original directory is NOT writable.
|
|||
// See https://github.com/otiai10/copy/pull/9 for more information.
|
|||
tmpPermissionForDirectory = os.FileMode(0755) |
|||
) |
|||
|
|||
// CopyAll copies src to dest, doesn't matter if src is a directory or a file.
|
|||
func CopyAll(vfs FileSystem, src, dest string, opt ...Options) error { |
|||
// FYI: os.Open does a proper lstat
|
|||
f, err := vfs.Open(src) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
info, err := f.Stat() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return switchboard(vfs, src, dest, f, info, assure(opt...)) |
|||
} |
|||
|
|||
// switchboard switches proper copy functions regarding file type, etc...
|
|||
// If there would be anything else here, add a case to this switchboard.
|
|||
func switchboard( |
|||
vfs FileSystem, src, dest string, f File, info os.FileInfo, opt Options, |
|||
) error { |
|||
switch { |
|||
case info.Mode()&os.ModeSymlink != 0: |
|||
// TODO
|
|||
return onsymlink(vfs, src, dest, opt) |
|||
case info.IsDir(): |
|||
return dcopy(vfs, src, dest, f, info, opt) |
|||
default: |
|||
return fcopy(vfs, src, dest, f, info, opt) |
|||
} |
|||
} |
|||
|
|||
// copy decide if this src should be copied or not.
|
|||
// Because this "copy" could be called recursively,
|
|||
// "info" MUST be given here, NOT nil.
|
|||
func copy(vfs FileSystem, src, dest string, f File, info os.FileInfo, opt Options) error { |
|||
skip, err := opt.Skip(src) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if skip { |
|||
return nil |
|||
} |
|||
return switchboard(vfs, src, dest, f, info, opt) |
|||
} |
|||
|
|||
// fcopy is for just a file,
|
|||
// with considering existence of parent directory
|
|||
// and file permission.
|
|||
func fcopy(vfs FileSystem, src, dest string, f File, info os.FileInfo, opt Options) (err error) { |
|||
|
|||
if err = os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { |
|||
return |
|||
} |
|||
|
|||
df, err := os.Create(dest) |
|||
if err != nil { |
|||
return |
|||
} |
|||
defer fclose(df, &err) |
|||
|
|||
if err = os.Chmod(df.Name(), info.Mode()|opt.AddPermission); err != nil { |
|||
return |
|||
} |
|||
|
|||
s, err := vfs.Open(src) |
|||
if err != nil { |
|||
return |
|||
} |
|||
defer fclose(s, &err) |
|||
|
|||
if _, err = io.Copy(df, s); err != nil { |
|||
return |
|||
} |
|||
|
|||
if opt.Sync { |
|||
err = df.Sync() |
|||
} |
|||
|
|||
return |
|||
} |
|||
|
|||
// dcopy is for a directory,
|
|||
// with scanning contents inside the directory
|
|||
// and pass everything to "copy" recursively.
|
|||
func dcopy(vfs FileSystem, srcdir, destdir string, d File, info os.FileInfo, opt Options) (err error) { |
|||
|
|||
originalMode := info.Mode() |
|||
|
|||
// Make dest dir with 0755 so that everything writable.
|
|||
if err = os.MkdirAll(destdir, tmpPermissionForDirectory); err != nil { |
|||
return |
|||
} |
|||
// Recover dir mode with original one.
|
|||
defer chmod(destdir, originalMode|opt.AddPermission, &err) |
|||
|
|||
fileInfos, err := d.Readdir(-1) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
for _, newInfo := range fileInfos { |
|||
cs, cd := filepath.Join( |
|||
srcdir, newInfo.Name()), |
|||
filepath.Join(destdir, newInfo.Name()) |
|||
|
|||
f, err := vfs.Open(cs) |
|||
if nil != err { |
|||
return err |
|||
} |
|||
if err := copy(vfs, cs, cd, f, newInfo, opt); err != nil { |
|||
// If any error, exit immediately
|
|||
return err |
|||
} |
|||
} |
|||
|
|||
return |
|||
} |
|||
|
|||
func onsymlink(vfs FileSystem, src, dest string, opt Options) error { |
|||
switch opt.OnSymlink(src) { |
|||
case Shallow: |
|||
return lcopy(vfs, src, dest) |
|||
/* |
|||
case Deep: |
|||
orig, err := vfs.EvalSymlinks(src) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
f, err := vfs.Open(orig) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
//info, err := os.Lstat(orig)
|
|||
info, err := f.Stat() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return copy(vfs, orig, dest, f, info, opt) |
|||
*/ |
|||
case Skip: |
|||
fallthrough |
|||
default: |
|||
return nil // do nothing
|
|||
} |
|||
} |
|||
|
|||
// lcopy is for a symlink,
|
|||
// with just creating a new symlink by replicating src symlink.
|
|||
func lcopy(vfs FileSystem, src, dest string) error { |
|||
src, err := vfs.Readlink(src) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
// Create the directories on the path to the dest symlink.
|
|||
if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { |
|||
return err |
|||
} |
|||
|
|||
return os.Symlink(src, dest) |
|||
} |
|||
|
|||
// fclose ANYHOW closes file,
|
|||
// with asiging error raised during Close,
|
|||
// BUT respecting the error already reported.
|
|||
func fclose(f File, reported *error) { |
|||
if err := f.Close(); *reported == nil { |
|||
*reported = err |
|||
} |
|||
} |
|||
|
|||
// chmod ANYHOW changes file mode,
|
|||
// with asiging error raised during Chmod,
|
|||
// BUT respecting the error already reported.
|
|||
func chmod(dir string, mode os.FileMode, reported *error) { |
|||
if err := os.Chmod(dir, mode); *reported == nil { |
|||
*reported = err |
|||
} |
|||
} |
|||
|
|||
// assure Options struct, should be called only once.
|
|||
// All optional values MUST NOT BE nil/zero after assured.
|
|||
func assure(opts ...Options) Options { |
|||
if len(opts) == 0 { |
|||
return getDefaultOptions() |
|||
} |
|||
defopt := getDefaultOptions() |
|||
if opts[0].OnSymlink == nil { |
|||
opts[0].OnSymlink = defopt.OnSymlink |
|||
} |
|||
if opts[0].Skip == nil { |
|||
opts[0].Skip = defopt.Skip |
|||
} |
|||
return opts[0] |
|||
} |
@ -0,0 +1,9 @@ |
|||
module git.rootprojects.org/root/vfscopy |
|||
|
|||
go 1.15 |
|||
|
|||
require ( |
|||
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect |
|||
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 |
|||
golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6 // indirect |
|||
) |
@ -0,0 +1,26 @@ |
|||
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= |
|||
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= |
|||
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU= |
|||
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= |
|||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= |
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= |
|||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
|||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= |
|||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= |
|||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= |
|||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
|||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
|||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6 h1:rbvTkL9AkFts1cgI78+gG6Yu1pwaqX6hjSJAatB78E4= |
|||
golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= |
|||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
@ -0,0 +1,44 @@ |
|||
package vfscopy |
|||
|
|||
import "os" |
|||
|
|||
// Options specifies optional actions on copying.
|
|||
type Options struct { |
|||
// OnSymlink can specify what to do on symlink
|
|||
OnSymlink func(src string) SymlinkAction |
|||
// Skip can specify which files should be skipped
|
|||
Skip func(src string) (bool, error) |
|||
// AddPermission to every entities,
|
|||
// NO MORE THAN 0777
|
|||
AddPermission os.FileMode |
|||
// Sync file after copy.
|
|||
// Useful in case when file must be on the disk
|
|||
// (in case crash happens, for example),
|
|||
// at the expense of some performance penalty
|
|||
Sync bool |
|||
} |
|||
|
|||
// SymlinkAction represents what to do on symlink.
|
|||
type SymlinkAction int |
|||
|
|||
const ( |
|||
// Shallow creates new symlink to the dest of symlink.
|
|||
Shallow SymlinkAction = iota |
|||
// Skip does nothing with symlink.
|
|||
Skip |
|||
) |
|||
|
|||
// getDefaultOptions provides default options,
|
|||
// which would be modified by usage-side.
|
|||
func getDefaultOptions() Options { |
|||
return Options{ |
|||
OnSymlink: func(string) SymlinkAction { |
|||
return Shallow // Do shallow copy
|
|||
}, |
|||
Skip: func(string) (bool, error) { |
|||
return false, nil // Don't skip
|
|||
}, |
|||
AddPermission: 0, // Add nothing
|
|||
Sync: false, // Do not sync
|
|||
} |
|||
} |
@ -0,0 +1,137 @@ |
|||
package vfscopy |
|||
|
|||
import ( |
|||
"os" |
|||
"io" |
|||
"io/ioutil" |
|||
"path/filepath" |
|||
"path" |
|||
"strings" |
|||
"errors" |
|||
"net/http" |
|||
) |
|||
|
|||
// FileSystem is a Virtual FileSystem with Symlink support
|
|||
type FileSystem interface { |
|||
Open(name string) (File, error) |
|||
Readlink(name string) (string, error) |
|||
//EvalSymlinks(name string) (string, error)
|
|||
} |
|||
|
|||
// File is copied from http.File
|
|||
type File interface { |
|||
io.Closer |
|||
io.Reader |
|||
io.Seeker |
|||
Readdir(count int) ([]os.FileInfo, error) |
|||
Stat() (os.FileInfo, error) |
|||
} |
|||
|
|||
// VFS is a virtual FileSystem with Symlink support
|
|||
type VFS struct { |
|||
FileSystem http.FileSystem |
|||
} |
|||
|
|||
// Open opens a file relative to a virtual filesystem
|
|||
func (v *VFS) Open(name string) (File, error) { |
|||
return v.FileSystem.Open(name) |
|||
} |
|||
|
|||
// Readlink returns a "not implemented" error,
|
|||
// which is okay because it is never called for http.FileSystem.
|
|||
func (v *VFS) Readlink(name string) (string, error) { |
|||
f, err := v.FileSystem.Open(name) |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
b, err := ioutil.ReadAll(f) |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
return string(b), nil |
|||
} |
|||
|
|||
// NewVFS gives an http.FileSystem (real) symlink support
|
|||
func NewVFS(httpfs http.FileSystem) FileSystem { |
|||
return &VFS{ FileSystem: httpfs } |
|||
} |
|||
|
|||
// Dir is an implementation of a Virtual FileSystem
|
|||
type Dir string |
|||
|
|||
// mapDirOpenError maps the provided non-nil error from opening name
|
|||
// to a possibly better non-nil error. In particular, it turns OS-specific errors
|
|||
// about opening files in non-directories into os.ErrNotExist. See Issue 18984.
|
|||
func mapDirOpenError(originalErr error, name string) error { |
|||
if os.IsNotExist(originalErr) || os.IsPermission(originalErr) { |
|||
return originalErr |
|||
} |
|||
|
|||
parts := strings.Split(name, string(filepath.Separator)) |
|||
for i := range parts { |
|||
if parts[i] == "" { |
|||
continue |
|||
} |
|||
fi, err := os.Stat(strings.Join(parts[:i+1], string(filepath.Separator))) |
|||
if err != nil { |
|||
return originalErr |
|||
} |
|||
if !fi.IsDir() { |
|||
return os.ErrNotExist |
|||
} |
|||
} |
|||
return originalErr |
|||
} |
|||
|
|||
func (d Dir) fullName(name string) (string, error) { |
|||
if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { |
|||
return "", errors.New("http: invalid character in file path") |
|||
} |
|||
dir := string(d) |
|||
if dir == "" { |
|||
dir = "." |
|||
} |
|||
fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) |
|||
return fullName, nil |
|||
} |
|||
|
|||
// Open opens a file relative to a virtual filesystem
|
|||
func (d Dir) Open(name string) (File, error) { |
|||
fullName, err := d.fullName(name) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
f, err := os.Open(fullName) |
|||
if err != nil { |
|||
return nil, mapDirOpenError(err, fullName) |
|||
} |
|||
return f, nil |
|||
} |
|||
|
|||
// Readlink returns the destination of the named symbolic link.
|
|||
func (d Dir) Readlink(name string) (string, error) { |
|||
name, err := d.fullName(name) |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
name, err = os.Readlink(name) |
|||
if err != nil { |
|||
return "", mapDirOpenError(err, name) |
|||
} |
|||
return name, nil |
|||
} |
|||
|
|||
/* |
|||
// EvalSymlinks returns the destination of the named symbolic link.
|
|||
func (d Dir) EvalSymlinks(name string) (string, error) { |
|||
name, err := d.fullName(name) |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
name, err = filepath.EvalSymlinks(name) |
|||
if err != nil { |
|||
return "", mapDirOpenError(err, name) |
|||
} |
|||
return name, nil |
|||
} |
|||
*/ |
Завантаження…
Посилання в новій проблемі