[upd] fix connent https vfs

8 months ago · 1a4b350eeb
8 changed files with 1287 additions and 1 deletions
--- a/pkg/s3/README.md
+++ b/pkg/s3/README.md
@ -0,0 +1,67 @@
+# S3 Stow Implementation
+
+Location = Amazon S3
+
+Container = Bucket
+
+Item = File
+
+Helpful Links:
+
+`http://docs.aws.amazon.com/sdk-for-go/api/service/s3/#example_S3_ListBuckets`
+
+---
+
+SDK Notes:
+
+- Metadata of an S3 Object can only be set when the Object is created.
+
+---
+
+Concerns:
+
+- An AWS account may have credentials which temporarily modifies permissions. This is specified by a token value. This feature is implemented but disabled and added as a TODO.
+
+---
+
+Things to know:
+
+- Paging for the list of containers doesn't exist yet, this is because there's a hard limit of about 100 containers for every account.
+
+- A client is required to provide a region. Manipulating buckets that reside within other regions isn't possible.
+
+---
+
+###### Dev Notes
+
+The init function of every implementation of `stow` must call `stow.Register`.
+
+`stow.Register` accepts a few things:
+
+### Kind, a string argument respresenting the name of the location.
+
+`makefn` a function that accepts any type that conforms to the stow.Config
+interface. It first validates the values of the `Config` argument, and then
+attempts to use the configuration to create a new client. If successful, An
+instance of a data type that conforms to the `stow.Location` interface is
+created. This Location should have fields that contain the client and
+configuration.
+
+Further calls in the hierarchy of a Location, Container, and Item depend
+on the values of the configuration + the client to send and receive information.
+
+- `kingmatchfn` a function that ensures that a given URL matches the `Kind` of the type of storage.
+
+---
+
+**stow.Register(kind string, makefn func(Config) (Locaion, error), kindmatchfn func(*url.URL) bool)**
+
+- Adds `kind` and `makefn` into a map that contains a list of locations.
+
+- Adds `kind` to a slice that contains all of the different kinds.
+
+- Adds `kind` as part of an anonymous function which validates the scheme of the url.URL
+
+Once the `stow.Register` function is completed, a location of the given kind is returned.
+
+---
--- a/pkg/s3/config.go
+++ b/pkg/s3/config.go
@ -0,0 +1,203 @@
+package s3
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/graymeta/stow"
+	"github.com/pkg/errors"
+)
+
+// Kind represents the name of the location/storage type.
+const Kind = "s3"
+
+var (
+	authTypeAccessKey = "accesskey"
+	authTypeIAM       = "iam"
+)
+
+const (
+	// ConfigAuthType is an optional argument that defines whether to use an IAM role or access key based auth
+	ConfigAuthType = "auth_type"
+
+	// ConfigAccessKeyID is one key of a pair of AWS credentials.
+	ConfigAccessKeyID = "access_key_id"
+
+	// ConfigSecretKey is one key of a pair of AWS credentials.
+	ConfigSecretKey = "secret_key"
+
+	// ConfigToken is an optional argument which is required when providing
+	// credentials with temporary access.
+	// ConfigToken = "token"
+
+	// ConfigRegion represents the region/availability zone of the session.
+	ConfigRegion = "region"
+
+	// ConfigEndpoint is optional config value for changing s3 endpoint
+	// used for e.g. minio.io
+	ConfigEndpoint = "endpoint"
+
+	// ConfigDisableSSL is optional config value for disabling SSL support on custom endpoints
+	// Its default value is "false", to disable SSL set it to "true".
+	ConfigDisableSSL = "disable_ssl"
+
+	// ConfigV2Signing is an optional config value for signing requests with the v2 signature.
+	// Its default value is "false", to enable set to "true".
+	// This feature is useful for s3-compatible blob stores -- ie minio.
+	ConfigV2Signing = "v2_signing"
+
+	ConfigCaCert = "ca_cert"
+)
+
+func init() {
+	validatefn := func(config stow.Config) error {
+		authType, ok := config.Config(ConfigAuthType)
+		if !ok || authType == "" {
+			authType = authTypeAccessKey
+		}
+
+		if !(authType == authTypeAccessKey || authType == authTypeIAM) {
+			return errors.New("invalid auth_type")
+		}
+
+		if authType == authTypeAccessKey {
+			_, ok := config.Config(ConfigAccessKeyID)
+			if !ok {
+				return errors.New("missing Access Key ID")
+			}
+
+			_, ok = config.Config(ConfigSecretKey)
+			if !ok {
+				return errors.New("missing Secret Key")
+			}
+		}
+		return nil
+	}
+	makefn := func(config stow.Config) (stow.Location, error) {
+
+		authType, ok := config.Config(ConfigAuthType)
+		if !ok || authType == "" {
+			authType = authTypeAccessKey
+		}
+
+		if !(authType == authTypeAccessKey || authType == authTypeIAM) {
+			return nil, errors.New("invalid auth_type")
+		}
+
+		if authType == authTypeAccessKey {
+			_, ok := config.Config(ConfigAccessKeyID)
+			if !ok {
+				return nil, errors.New("missing Access Key ID")
+			}
+
+			_, ok = config.Config(ConfigSecretKey)
+			if !ok {
+				return nil, errors.New("missing Secret Key")
+			}
+		}
+
+		// Create a new client (s3 session)
+		client, endpoint, err := newS3Client(config, "")
+		if err != nil {
+			return nil, err
+		}
+
+		// Create a location with given config and client (s3 session).
+		loc := &location{
+			config:         config,
+			client:         client,
+			customEndpoint: endpoint,
+		}
+
+		return loc, nil
+	}
+
+	kindfn := func(u *url.URL) bool {
+		return u.Scheme == Kind
+	}
+
+	stow.Register(Kind, makefn, kindfn, validatefn)
+}
+
+// Attempts to create a session based on the information given.
+func newS3Client(config stow.Config, region string) (client *s3.S3, endpoint string, err error) {
+	authType, _ := config.Config(ConfigAuthType)
+	accessKeyID, _ := config.Config(ConfigAccessKeyID)
+	secretKey, _ := config.Config(ConfigSecretKey)
+	//	token, _ := config.Config(ConfigToken)
+	caCert, _ := config.Config(ConfigCaCert)
+
+	if authType == "" {
+		authType = authTypeAccessKey
+	}
+
+	httpClient := http.DefaultClient
+
+	if caCert != "" {
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM([]byte(caCert))
+
+		transport := &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caCertPool,
+			},
+		}
+
+		httpClient.Transport = transport
+	}
+
+	awsConfig := aws.NewConfig().
+		WithHTTPClient(httpClient).
+		WithMaxRetries(aws.UseServiceDefaultRetries).
+		WithLogger(aws.NewDefaultLogger()).
+		WithLogLevel(aws.LogOff).
+		WithSleepDelay(time.Sleep)
+
+	if region == "" {
+		region, _ = config.Config(ConfigRegion)
+	}
+	if region != "" {
+		awsConfig.WithRegion(region)
+	} else {
+		awsConfig.WithRegion("us-east-1")
+	}
+
+	if authType == authTypeAccessKey {
+		awsConfig.WithCredentials(credentials.NewStaticCredentials(accessKeyID, secretKey, ""))
+	}
+
+	endpoint, ok := config.Config(ConfigEndpoint)
+	if ok {
+		awsConfig.WithEndpoint(endpoint).
+			WithS3ForcePathStyle(true)
+	}
+
+	disableSSL, ok := config.Config(ConfigDisableSSL)
+	if ok && disableSSL == "true" {
+		awsConfig.WithDisableSSL(true)
+	}
+
+	sess, err := session.NewSession(awsConfig)
+	if err != nil {
+		return nil, "", err
+	}
+	if sess == nil {
+		return nil, "", errors.New("creating the S3 session")
+	}
+
+	s3Client := s3.New(sess)
+
+	usev2, ok := config.Config(ConfigV2Signing)
+	if ok && usev2 == "true" {
+		setv2Handlers(s3Client)
+	}
+
+	return s3Client, endpoint, nil
+}
--- a/pkg/s3/container.go
+++ b/pkg/s3/container.go
@ -0,0 +1,268 @@
+package s3
+
+import (
+	"io"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/graymeta/stow"
+	"github.com/pkg/errors"
+)
+
+// Amazon S3 bucket contains a creation date and a name.
+type container struct {
+	// name is needed to retrieve items.
+	name string
+	// client is responsible for performing the requests.
+	client *s3.S3
+	// region describes the AWS Availability Zone of the S3 Bucket.
+	region         string
+	customEndpoint string
+}
+
+// ID returns a string value which represents the name of the container.
+func (c *container) ID() string {
+	return c.name
+}
+
+// Name returns a string value which represents the name of the container.
+func (c *container) Name() string {
+	return c.name
+}
+
+// Item returns a stow.Item instance of a container based on the name of the container and the key representing. The
+// retrieved item only contains metadata about the object. This ensures that only the minimum amount of information is
+// transferred. Calling item.Open() will actually do a get request and open a stream to read from.
+func (c *container) Item(id string) (stow.Item, error) {
+	return c.getItem(id)
+}
+
+// Items sends a request to retrieve a list of items that are prepended with
+// the prefix argument. The 'cursor' variable facilitates pagination.
+func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) {
+	itemLimit := int64(count)
+
+	params := &s3.ListObjectsV2Input{
+		Bucket:     aws.String(c.Name()),
+		StartAfter: &cursor,
+		MaxKeys:    &itemLimit,
+		Prefix:     &prefix,
+	}
+
+	response, err := c.client.ListObjectsV2(params)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "Items, listing objects")
+	}
+
+	var containerItems []stow.Item
+
+	for _, object := range response.Contents {
+		if *object.StorageClass == "GLACIER" {
+			continue
+		}
+		etag := cleanEtag(*object.ETag) // Copy etag value and remove the strings.
+		object.ETag = &etag             // Assign the value to the object field representing the item.
+
+		newItem := &item{
+			container: c,
+			client:    c.client,
+			properties: properties{
+				ETag:         object.ETag,
+				Key:          object.Key,
+				LastModified: object.LastModified,
+				Owner:        object.Owner,
+				Size:         object.Size,
+				StorageClass: object.StorageClass,
+			},
+		}
+		containerItems = append(containerItems, newItem)
+	}
+
+	// Create a marker and determine if the list of items to retrieve is complete.
+	// If not, the last file is the input to the value of after which item to start
+	startAfter := ""
+	if *response.IsTruncated {
+		startAfter = containerItems[len(containerItems)-1].Name()
+	}
+
+	return containerItems, startAfter, nil
+}
+
+func (c *container) RemoveItem(id string) error {
+	params := &s3.DeleteObjectInput{
+		Bucket: aws.String(c.Name()),
+		Key:    aws.String(id),
+	}
+
+	_, err := c.client.DeleteObject(params)
+	if err != nil {
+		return errors.Wrapf(err, "RemoveItem, deleting object %+v", params)
+	}
+	return nil
+}
+
+// Put sends a request to upload content to the container. The arguments
+// received are the name of the item (S3 Object), a reader representing the
+// content, and the size of the file. Many more attributes can be given to the
+// file, including metadata. Keeping it simple for now.
+func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) {
+	// Convert map[string]interface{} to map[string]*string
+	mdPrepped, err := prepMetadata(metadata)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to create or update item, preparing metadata")
+	}
+
+	uploader := s3manager.NewUploaderWithClient(c.client)
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket:   aws.String(c.name), // Required
+		Key:      aws.String(name),   // Required
+		Body:     r,
+		Metadata: mdPrepped, // map[string]*string
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "PutObject, putting object")
+	}
+	i, err := c.client.HeadObject(&s3.HeadObjectInput{
+		Key:    aws.String(name),
+		Bucket: aws.String(c.name),
+	})
+	var etag string
+	if i.ETag != nil && err == nil {
+		etag = cleanEtag(*i.ETag)
+	}
+
+	// Some fields are empty because this information isn't included in the response.
+	// May have to involve sending a request if we want more specific information.
+	// Keeping it simple for now.
+	// s3.Object info: https://github.com/aws/aws-sdk-go/blob/master/service/s3/api.go#L7092-L7107
+	// Response: https://github.com/aws/aws-sdk-go/blob/master/service/s3/api.go#L8193-L8227
+	newItem := &item{
+		container: c,
+		client:    c.client,
+		properties: properties{
+			ETag: &etag,
+			Key:  &name,
+			Size: &size,
+			//LastModified *time.Time
+			//Owner        *s3.Owner
+			//StorageClass *string
+		},
+	}
+
+	return newItem, nil
+}
+
+// Region returns a string representing the region/availability zone of the container.
+func (c *container) Region() string {
+	return c.region
+}
+
+// A request to retrieve a single item includes information that is more specific than
+// a PUT. Instead of doing a request within the PUT, make this method available so that the
+// request can be made by the field retrieval methods when necessary. This is the case for
+// fields that are left out, such as the object's last modified date. This also needs to be
+// done only once since the requested information is retained.
+// May be simpler to just stick it in PUT and and do a request every time, please vouch
+// for this if so.
+func (c *container) getItem(id string) (*item, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(c.name),
+		Key:    aws.String(id),
+	}
+
+	res, err := c.client.HeadObject(params)
+	if err != nil {
+		// stow needs ErrNotFound to pass the test but amazon returns an opaque error
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+			return nil, stow.ErrNotFound
+		}
+		return nil, errors.Wrap(err, "getItem, getting the object")
+	}
+
+	etag := cleanEtag(*res.ETag) // etag string value contains quotations. Remove them.
+	md, err := parseMetadata(res.Metadata)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to retrieve Item information, parsing metadata")
+	}
+
+	i := &item{
+		container: c,
+		client:    c.client,
+		properties: properties{
+			ETag:         &etag,
+			Key:          &id,
+			LastModified: res.LastModified,
+			Owner:        nil, // not returned in the response.
+			Size:         res.ContentLength,
+			StorageClass: res.StorageClass,
+			Metadata:     md,
+		},
+	}
+
+	return i, nil
+}
+
+// Remove quotation marks from beginning and end. This includes quotations that
+// are escaped. Also removes leading `W/` from prefix for weak Etags.
+//
+// Based on the Etag spec, the full etag value (<FULL ETAG VALUE>) can include:
+// - W/"<ETAG VALUE>"
+// - "<ETAG VALUE>"
+// - ""
+// Source: https://tools.ietf.org/html/rfc7232#section-2.3
+//
+// Based on HTTP spec, forward slash is a separator and must be enclosed in
+// quotes to be used as a valid value. Hence, the returned value may include:
+// - "<FULL ETAG VALUE>"
+// - \"<FULL ETAG VALUE>\"
+// Source: https://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
+//
+// This function contains a loop to check for the presence of the three possible
+// filler characters and strips them, resulting in only the Etag value.
+func cleanEtag(etag string) string {
+	for {
+		// Check if the filler characters are present
+		if strings.HasPrefix(etag, `\"`) {
+			etag = strings.Trim(etag, `\"`)
+
+		} else if strings.HasPrefix(etag, `"`) {
+			etag = strings.Trim(etag, `"`)
+
+		} else if strings.HasPrefix(etag, `W/`) {
+			etag = strings.Replace(etag, `W/`, "", 1)
+
+		} else {
+			break
+		}
+	}
+	return etag
+}
+
+// prepMetadata parses a raw map into the native type required by S3 to set metadata (map[string]*string).
+// TODO: validation for key values. This function also assumes that the value of a key value pair is a string.
+func prepMetadata(md map[string]interface{}) (map[string]*string, error) {
+	m := make(map[string]*string, len(md))
+	for key, value := range md {
+		strValue, valid := value.(string)
+		if !valid {
+			return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key)
+		}
+		m[key] = aws.String(strValue)
+	}
+	return m, nil
+}
+
+// The first letter of a dash separated key value is capitalized, so perform a ToLower on it.
+// This Key transformation of returning lowercase is consistent with other locations..
+func parseMetadata(md map[string]*string) (map[string]interface{}, error) {
+	m := make(map[string]interface{}, len(md))
+	for key, value := range md {
+		k := strings.ToLower(key)
+		m[k] = *value
+	}
+	return m, nil
+}
--- a/pkg/s3/doc.go
+++ b/pkg/s3/doc.go
@ -0,0 +1,44 @@
+/*
+Package s3 provides an abstraction of Amazon S3 (Simple Storage Service). An S3 Bucket is a Stow Container and an S3 Object is a Stow Item. Recall that nested directories exist within S3.
+
+Usage and Credentials
+
+There are three separate pieces of information required by Stow to have access to an S3 Stow Location: an AWS User's ACCESS_KEY_ID and SECRET_KEY fields, as well as the physical region of the S3 Endpoint. Ensure that the AWS User whose credentials are used to manipulate the S3 endpoint has permissions to do so.
+
+stow.Dial requires both a string value ("s3") of the particular Stow Location Kind and a stow.Config instance. The stow.Config instance requires three entries with the specific key value attributes:
+
+- a key of s3.ConfigAccessKeyID with a value of the AWS account's Access Key ID
+- a key of s3.ConfigSecretKey with a value of the AWS account's Secret Key
+- a key of s3.ConfigRegion with a value of the S3 endpoint's region (in all lowercase)
+
+Location
+
+The s3.location methods allow the retrieval of an S3 endpoint's Bucket or list of Buckets (Container or Containers). A stow.Item representation of an S3 Object can also be retrieved based on the Object's URL (ItemByURL).
+
+Additional s3.location methods provide capabilities to create and remove S3 Buckets (CreateContainer or RemoveContainer, respectively).
+
+Container
+
+There are s3.container methods which can retrieve an S3 Bucket's:
+
+- name (ID or Name)
+- Object or complete list of Objects (Item or Items)
+- region
+
+Additional s3.container methods give Stow the ability to:
+
+- remove an S3 Bucket (RemoveItem)
+- update or create an S3 Object (Put)
+
+Item
+
+Methods within an s3.item allow the retrieval of an S3 Object's:
+- name (ID or name)
+- URL (ItemByUrl)
+- size in bytes (Size)
+- S3 specific metadata (Metadata, key value pairs usually found within the console)
+- last modified date (LastMod)
+- Etag (Etag)
+- content (Open)
+*/
+package s3
--- a/pkg/s3/item.go
+++ b/pkg/s3/item.go
@ -0,0 +1,202 @@
+package s3
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/graymeta/stow"
+	"github.com/pkg/errors"
+)
+
+// The item struct contains an id (also the name of the file/S3 Object/Item),
+// a container which it belongs to (s3 Bucket), a client, and a URL. The last
+// field, properties, contains information about the item, including the ETag,
+// file name/id, size, owner, last modified date, and storage class.
+// see Object type at http://docs.aws.amazon.com/sdk-for-go/api/service/s3/
+// for more info.
+// All fields are unexported because methods exist to facilitate retrieval.
+type item struct {
+	// Container information is required by a few methods.
+	container *container
+	// A client is needed to make requests.
+	client *s3.S3
+	// properties represent the characteristics of the file. Name, Etag, etc.
+	properties properties
+	infoOnce   sync.Once
+	infoErr    error
+	tags       map[string]interface{}
+	tagsOnce   sync.Once
+	tagsErr    error
+}
+
+type properties struct {
+	ETag         *string    `type:"string"`
+	Key          *string    `min:"1" type:"string"`
+	LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"`
+	Owner        *s3.Owner  `type:"structure"`
+	Size         *int64     `type:"integer"`
+	StorageClass *string    `type:"string" enum:"ObjectStorageClass"`
+	Metadata     map[string]interface{}
+}
+
+// ID returns a string value that represents the name of a file.
+func (i *item) ID() string {
+	return *i.properties.Key
+}
+
+// Name returns a string value that represents the name of the file.
+func (i *item) Name() string {
+	return *i.properties.Key
+}
+
+// Size returns the size of an item in bytes.
+func (i *item) Size() (int64, error) {
+	return *i.properties.Size, nil
+}
+
+// URL returns a formatted string which follows the predefined format
+// that every S3 asset is given.
+func (i *item) URL() *url.URL {
+	if i.container.customEndpoint == "" {
+		genericURL := fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", i.container.Region(), i.container.Name(), i.Name())
+
+		return &url.URL{
+			Scheme: "s3",
+			Path:   genericURL,
+		}
+	}
+
+	genericURL := fmt.Sprintf("%s/%s", i.container.Name(), i.Name())
+	return &url.URL{
+		Scheme: "s3",
+		Path:   genericURL,
+	}
+}
+
+// Open retrieves specic information about an item based on the container name
+// and path of the file within the container. This response includes the body of
+// resource which is returned along with an error.
+func (i *item) Open() (io.ReadCloser, error) {
+	params := &s3.GetObjectInput{
+		Bucket: aws.String(i.container.Name()),
+		Key:    aws.String(i.ID()),
+	}
+
+	response, err := i.client.GetObject(params)
+	if err != nil {
+		return nil, errors.Wrap(err, "Open, getting the object")
+	}
+	return response.Body, nil
+}
+
+// LastMod returns the last modified date of the item. The response of an item that is PUT
+// does not contain this field. Solution? Detect when the LastModified field (a *time.Time)
+// is nil, then do a manual request for it via the Item() method of the container which
+// does return the specified field. This more detailed information is kept so that we
+// won't have to do it again.
+func (i *item) LastMod() (time.Time, error) {
+	err := i.ensureInfo()
+	if err != nil {
+		return time.Time{}, errors.Wrap(err, "retrieving Last Modified information of Item")
+	}
+	return *i.properties.LastModified, nil
+}
+
+// ETag returns the ETag value from the properies field of an item.
+func (i *item) ETag() (string, error) {
+	return *(i.properties.ETag), nil
+}
+
+func (i *item) Metadata() (map[string]interface{}, error) {
+	err := i.ensureInfo()
+	if err != nil {
+		return nil, errors.Wrap(err, "retrieving metadata")
+	}
+	return i.properties.Metadata, nil
+}
+
+func (i *item) ensureInfo() error {
+	if i.properties.Metadata == nil || i.properties.LastModified == nil {
+		i.infoOnce.Do(func() {
+			// Retrieve Item information
+			itemInfo, infoErr := i.getInfo()
+			if infoErr != nil {
+				i.infoErr = infoErr
+				return
+			}
+
+			// Set metadata field
+			i.properties.Metadata, infoErr = itemInfo.Metadata()
+			if infoErr != nil {
+				i.infoErr = infoErr
+				return
+			}
+
+			// Set LastModified field
+			lmValue, infoErr := itemInfo.LastMod()
+			if infoErr != nil {
+				i.infoErr = infoErr
+				return
+			}
+			i.properties.LastModified = &lmValue
+		})
+	}
+	return i.infoErr
+}
+
+func (i *item) getInfo() (stow.Item, error) {
+	itemInfo, err := i.container.getItem(i.ID())
+	if err != nil {
+		return nil, err
+	}
+	return itemInfo, nil
+}
+
+// Tags returns a map of tags on an Item
+func (i *item) Tags() (map[string]interface{}, error) {
+	i.tagsOnce.Do(func() {
+		params := &s3.GetObjectTaggingInput{
+			Bucket: aws.String(i.container.name),
+			Key:    aws.String(i.ID()),
+		}
+
+		res, err := i.client.GetObjectTagging(params)
+		if err != nil {
+			if strings.Contains(err.Error(), "NoSuchKey") {
+				i.tagsErr = stow.ErrNotFound
+				return
+			}
+			i.tagsErr = errors.Wrap(err, "getObjectTagging")
+			return
+		}
+
+		i.tags = make(map[string]interface{})
+		for _, t := range res.TagSet {
+			i.tags[*t.Key] = *t.Value
+		}
+	})
+
+	return i.tags, i.tagsErr
+}
+
+// OpenRange opens the item for reading starting at byte start and ending
+// at byte end.
+func (i *item) OpenRange(start, end uint64) (io.ReadCloser, error) {
+	params := &s3.GetObjectInput{
+		Bucket: aws.String(i.container.Name()),
+		Key:    aws.String(i.ID()),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
+	}
+
+	response, err := i.client.GetObject(params)
+	if err != nil {
+		return nil, errors.Wrap(err, "Open, getting the object")
+	}
+	return response.Body, nil
+}
--- a/pkg/s3/location.go
+++ b/pkg/s3/location.go
@ -0,0 +1,266 @@
+package s3
+
+import (
+	"context"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/graymeta/stow"
+	"github.com/pkg/errors"
+)
+
+// A location contains a client + the configurations used to create the client.
+type location struct {
+	config         stow.Config
+	customEndpoint string
+	client         *s3.S3
+}
+
+// CreateContainer creates a new container, in this case an S3 bucket.
+// The bare minimum needed is a container name, but there are many other
+// options that can be provided.
+func (l *location) CreateContainer(containerName string) (stow.Container, error) {
+	createBucketParams := &s3.CreateBucketInput{
+		Bucket: aws.String(containerName), // required
+	}
+
+	_, err := l.client.CreateBucket(createBucketParams)
+	if err != nil {
+		return nil, errors.Wrap(err, "CreateContainer, creating the bucket")
+	}
+
+	region, _ := l.config.Config("region")
+
+	newContainer := &container{
+		name:           containerName,
+		client:         l.client,
+		region:         region,
+		customEndpoint: l.customEndpoint,
+	}
+
+	return newContainer, nil
+}
+
+// Containers returns a slice of the Container interface, a cursor, and an error.
+// This doesn't seem to exist yet in the API without doing a ton of manual work.
+// Get the list of buckets, query every single one to retrieve region info, and finally
+// return the list of containers that have a matching region against the client. It's not
+// possible to manipulate a container within a region that doesn't match the clients'.
+// This is because AWS user credentials can be tied to regions. One solution would be
+// to start a new client for every single container where the region matches, this would
+// also check the credentials on every new instance... Tabled for later.
+func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) {
+	// Response returns exported Owner(*s3.Owner) and Bucket(*s3.[]Bucket)
+	var params *s3.ListBucketsInput
+	bucketList, err := l.client.ListBuckets(params)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "Containers, listing the buckets")
+	}
+
+	// Seek to the current bucket, according to cursor.
+	if cursor != stow.CursorStart {
+		ok := false
+		for i, b := range bucketList.Buckets {
+			if *b.Name == cursor {
+				ok = true
+				bucketList.Buckets = bucketList.Buckets[i:]
+				break
+			}
+		}
+		if !ok {
+			return nil, "", stow.ErrBadCursor
+		}
+	}
+	cursor = ""
+
+	// Region is pulled from stow.Config. If Region is specified, only add
+	// Bucket to Container list if it is located in configured Region.
+	region, regionSet := l.config.Config(ConfigRegion)
+
+	// Endpoint would indicate that we are using s3-compatible storage, which
+	// does not support s3session.GetBucketRegion().
+	endpoint, endpointSet := l.config.Config(ConfigEndpoint)
+
+	// Iterate through the slice of pointers to buckets
+	var containers []stow.Container
+	for _, bucket := range bucketList.Buckets {
+		if len(containers) == count {
+			cursor = *bucket.Name
+			break
+		}
+
+		if !strings.HasPrefix(*bucket.Name, prefix) {
+			continue
+		}
+
+		var err error
+		client := l.client
+		bucketRegion := region
+		if !endpointSet && endpoint == "" {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			bucketRegion, err = s3manager.GetBucketRegionWithClient(ctx, l.client, *bucket.Name)
+			cancel()
+			if err != nil {
+				if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+					// sometimes buckets will still show up int eh ListBuckets results after
+					// being deleted, but will 404 when determining the region. Use this as a
+					// strong signal that the bucket has been deleted.
+					continue
+				}
+				return nil, "", errors.Wrapf(err, "Containers, getting bucket region for: %s", *bucket.Name)
+			}
+			if regionSet && region != "" && bucketRegion != region {
+				continue
+			}
+
+			client, _, err = newS3Client(l.config, bucketRegion)
+			if err != nil {
+				return nil, "", errors.Wrapf(err, "Containers, creating new client for region: %s", bucketRegion)
+			}
+		}
+
+		newContainer := &container{
+			name:           *(bucket.Name),
+			client:         client,
+			region:         bucketRegion,
+			customEndpoint: l.customEndpoint,
+		}
+
+		containers = append(containers, newContainer)
+	}
+
+	return containers, cursor, nil
+}
+
+// Close simply satisfies the Location interface. There's nothing that
+// needs to be done in order to satisfy the interface.
+func (l *location) Close() error {
+	return nil // nothing to close
+}
+
+// Container retrieves a stow.Container based on its name which must be
+// exact.
+func (l *location) Container(id string) (stow.Container, error) {
+	client := l.client
+	bucketRegion, bucketRegionSet := l.config.Config(ConfigRegion)
+
+	// Endpoint would indicate that we are using s3-compatible storage, which
+	// does not support s3session.GetBucketRegion().
+	if endpoint, endpointSet := l.config.Config(ConfigEndpoint); !endpointSet && endpoint == "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		bucketRegion, _ = s3manager.GetBucketRegionWithClient(ctx, l.client, id)
+		cancel()
+
+		var err error
+		client, _, err = newS3Client(l.config, bucketRegion)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Container, creating new client for region: %s", bucketRegion)
+		}
+	}
+
+	c := &container{
+		name:           id,
+		client:         client,
+		region:         bucketRegion,
+		customEndpoint: l.customEndpoint,
+	}
+
+	if bucketRegionSet || bucketRegion != "" {
+		return c, nil
+	}
+
+	params := &s3.GetBucketLocationInput{
+		Bucket: aws.String(id),
+	}
+
+	_, err := client.GetBucketLocation(params)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoSuchBucket" {
+			return nil, stow.ErrNotFound
+		}
+
+		return nil, errors.Wrap(err, "GetBucketLocation")
+	}
+
+	return c, nil
+}
+
+// RemoveContainer removes a container simply by name.
+func (l *location) RemoveContainer(id string) error {
+	params := &s3.DeleteBucketInput{
+		Bucket: aws.String(id),
+	}
+
+	_, err := l.client.DeleteBucket(params)
+	if err != nil {
+		return errors.Wrap(err, "RemoveContainer, deleting the bucket")
+	}
+
+	return nil
+}
+
+// ItemByURL retrieves a stow.Item by parsing the URL, in this
+// case an item is an object.
+func (l *location) ItemByURL(url *url.URL) (stow.Item, error) {
+	if l.customEndpoint == "" {
+		genericURL := []string{"https://s3-", ".amazonaws.com/"}
+
+		// Remove genericURL[0] from URL:
+		// url = <genericURL[0]><region><genericURL[1]><bucket name><object path>
+		firstCut := strings.Replace(url.Path, genericURL[0], "", 1)
+
+		// find first dot so that we could extract region.
+		dotIndex := strings.Index(firstCut, ".")
+
+		// region of the s3 bucket.
+		region := firstCut[0:dotIndex]
+
+		// Remove <region><genericURL[1]> from
+		// <region><genericURL[1]><bucket name><object path>
+		secondCut := strings.Replace(firstCut, region+genericURL[1], "", 1)
+
+		// Get the index of the first slash to get the end of the bucket name.
+		firstSlash := strings.Index(secondCut, "/")
+
+		// Grab bucket name
+		bucketName := secondCut[:firstSlash]
+
+		// Everything afterwards pertains to object.
+		objectPath := secondCut[firstSlash+1:]
+
+		// Get the container by bucket name.
+		cont, err := l.Container(bucketName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "ItemByURL, getting container by the bucketname %s", bucketName)
+		}
+
+		// Get the item by object name.
+		it, err := cont.Item(objectPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "ItemByURL, getting item by object name %s", objectPath)
+		}
+
+		return it, err
+	}
+
+	// url looks like this: s3://<containerName>/<itemName>
+	// example: s3://graymeta-demo/DPtest.txt
+	containerName := url.Host
+	itemName := strings.TrimPrefix(url.Path, "/")
+
+	c, err := l.Container(containerName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "ItemByURL, getting container by the bucketname %s", containerName)
+	}
+
+	i, err := c.Item(itemName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "ItemByURL, getting item by object name %s", itemName)
+	}
+	return i, nil
+}
--- a/pkg/s3/v2signer.go
+++ b/pkg/s3/v2signer.go
@ -0,0 +1,234 @@
+/*
+Copyright (c) 2013 Damien Le Berrigaud and Nick Wade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package s3
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/corehandlers"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const (
+	signatureVersion = "2"
+	signatureMethod  = "HmacSHA1"
+	timeFormat       = "2006-01-02T15:04:05Z"
+)
+
+type signer struct {
+	// Values that must be populated from the request
+	Request     *http.Request
+	Time        time.Time
+	Credentials *credentials.Credentials
+	Debug       aws.LogLevelType
+	Logger      aws.Logger
+
+	Query        url.Values
+	stringToSign string
+	signature    string
+}
+
+var s3ParamsToSign = map[string]bool{
+	"acl":                          true,
+	"location":                     true,
+	"logging":                      true,
+	"notification":                 true,
+	"partNumber":                   true,
+	"policy":                       true,
+	"requestPayment":               true,
+	"torrent":                      true,
+	"uploadId":                     true,
+	"uploads":                      true,
+	"versionId":                    true,
+	"versioning":                   true,
+	"versions":                     true,
+	"response-content-type":        true,
+	"response-content-language":    true,
+	"response-expires":             true,
+	"response-cache-control":       true,
+	"response-content-disposition": true,
+	"response-content-encoding":    true,
+	"website":                      true,
+	"delete":                       true,
+}
+
+func setv2Handlers(svc *s3.S3) {
+	svc.Handlers.Build.PushBack(func(r *request.Request) {
+		parsedURL, err := url.Parse(r.HTTPRequest.URL.String())
+		if err != nil {
+			log.Fatal("Failed to parse URL", err)
+		}
+		r.HTTPRequest.URL.Opaque = parsedURL.Path
+	})
+
+	svc.Handlers.Sign.Clear()
+	svc.Handlers.Sign.PushBack(Sign)
+	svc.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+}
+
+// Sign requests with signature version 2.
+//
+// Will sign the requests with the service config's Credentials object
+// Signing is skipped if the credentials is the credentials.AnonymousCredentials
+// object.
+func Sign(req *request.Request) {
+	// If the request does not need to be signed ignore the signing of the
+	// request if the AnonymousCredentials object is used.
+	if req.Config.Credentials == credentials.AnonymousCredentials {
+		return
+	}
+
+	v2 := signer{
+		Request:     req.HTTPRequest,
+		Time:        req.Time,
+		Credentials: req.Config.Credentials,
+		Debug:       req.Config.LogLevel.Value(),
+		Logger:      req.Config.Logger,
+	}
+
+	req.Error = v2.Sign()
+
+	if req.Error != nil {
+		return
+	}
+}
+
+func (v2 *signer) Sign() error {
+	credValue, err := v2.Credentials.Get()
+	if err != nil {
+		return err
+	}
+	accessKey := credValue.AccessKeyID
+	var (
+		md5, ctype, date, xamz string
+		xamzDate               bool
+		sarray                 []string
+	)
+
+	headers := v2.Request.Header
+	params := v2.Request.URL.Query()
+	parsedURL, err := url.Parse(v2.Request.URL.String())
+	if err != nil {
+		return err
+	}
+	host, canonicalPath := parsedURL.Host, parsedURL.Path
+	v2.Request.Header["Host"] = []string{host}
+	v2.Request.Header["x-amz-date"] = []string{v2.Time.In(time.UTC).Format(time.RFC1123)}
+
+	for k, v := range headers {
+		k = strings.ToLower(k)
+		switch k {
+		case "content-md5":
+			md5 = v[0]
+		case "content-type":
+			ctype = v[0]
+		case "date":
+			if !xamzDate {
+				date = v[0]
+			}
+		default:
+			if strings.HasPrefix(k, "x-amz-") {
+				vall := strings.Join(v, ",")
+				sarray = append(sarray, k+":"+vall)
+				if k == "x-amz-date" {
+					xamzDate = true
+					date = ""
+				}
+			}
+		}
+	}
+	if len(sarray) > 0 {
+		sort.StringSlice(sarray).Sort()
+		xamz = strings.Join(sarray, "\n") + "\n"
+	}
+
+	expires := false
+	if v, ok := params["Expires"]; ok {
+		expires = true
+		date = v[0]
+		params["AWSAccessKeyId"] = []string{accessKey}
+	}
+
+	sarray = sarray[0:0]
+	for k, v := range params {
+		if s3ParamsToSign[k] {
+			for _, vi := range v {
+				if vi == "" {
+					sarray = append(sarray, k)
+				} else {
+					sarray = append(sarray, k+"="+vi)
+				}
+			}
+		}
+	}
+	if len(sarray) > 0 {
+		sort.StringSlice(sarray).Sort()
+		canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&")
+	}
+
+	v2.stringToSign = strings.Join([]string{
+		v2.Request.Method,
+		md5,
+		ctype,
+		date,
+		xamz + canonicalPath,
+	}, "\n")
+	hash := hmac.New(sha1.New, []byte(credValue.SecretAccessKey))
+	hash.Write([]byte(v2.stringToSign))
+	v2.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))
+
+	if expires {
+		params["Signature"] = []string{string(v2.signature)}
+	} else {
+		headers["Authorization"] = []string{"AWS " + accessKey + ":" + string(v2.signature)}
+	}
+
+	if v2.Debug.Matches(aws.LogDebugWithSigning) {
+		v2.logSigningInfo()
+	}
+	return nil
+}
+
+const logSignInfoMsg = `DEBUG: Request Signature:
+---[ STRING TO SIGN ]--------------------------------
+%s
+---[ SIGNATURE ]-------------------------------------
+%s
+-----------------------------------------------------`
+
+func (v2 *signer) logSigningInfo() {
+	msg := fmt.Sprintf(logSignInfoMsg, v2.stringToSign, v2.signature)
+	v2.Logger.Log(msg)
+}
--- a/vfs.go
+++ b/vfs.go
@ -10,10 +10,10 @@ import (
 	"net/url"
 	"strings"

+	"git.lowcodeplatform.net/fabric/lib/pkg/s3"
 	"github.com/graymeta/stow"
 	"github.com/graymeta/stow/azure"
 	"github.com/graymeta/stow/local"
-	"github.com/graymeta/stow/s3"

 	// support Azure storage
 	_ "github.com/graymeta/stow/azure"
@ -35,6 +35,7 @@ type vfs struct {
 	location                                       stow.Location
 	container                                      stow.Container
 	comma                                          string
+	cacert                                         string
 }

 type Vfs interface {
@ -67,6 +68,7 @@ func (v *vfs) Connect() (err error) {
 			s3.ConfigAccessKeyID: v.accessKeyID,
 			s3.ConfigSecretKey:   v.secretKey,
 			s3.ConfigRegion:      v.region,
+			s3.ConfigCaCert:      v.cacert,
 		}
 	case "azure":
 		config = stow.ConfigMap{