// Copyright 2015 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bigquery import ( "errors" "reflect" bq "google.golang.org/api/bigquery/v2" ) // Schema describes the fields in a table or query result. type Schema []*FieldSchema type FieldSchema struct { // The field name. // Must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_), // and must start with a letter or underscore. // The maximum length is 128 characters. Name string // A description of the field. The maximum length is 16,384 characters. Description string // Whether the field may contain multiple values. Repeated bool // Whether the field is required. Ignored if Repeated is true. Required bool // The field data type. If Type is Record, then this field contains a nested schema, // which is described by Schema. Type FieldType // Describes the nested schema if Type is set to Record. Schema Schema } func (fs *FieldSchema) asTableFieldSchema() *bq.TableFieldSchema { tfs := &bq.TableFieldSchema{ Description: fs.Description, Name: fs.Name, Type: string(fs.Type), } if fs.Repeated { tfs.Mode = "REPEATED" } else if fs.Required { tfs.Mode = "REQUIRED" } // else leave as default, which is interpreted as NULLABLE. for _, f := range fs.Schema { tfs.Fields = append(tfs.Fields, f.asTableFieldSchema()) } return tfs } func (s Schema) asTableSchema() *bq.TableSchema { var fields []*bq.TableFieldSchema for _, f := range s { fields = append(fields, f.asTableFieldSchema()) } return &bq.TableSchema{Fields: fields} } // customizeCreateTable allows a Schema to be used directly as an option to CreateTable. func (s Schema) customizeCreateTable(conf *createTableConf) { conf.schema = s.asTableSchema() } func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema { fs := &FieldSchema{ Description: tfs.Description, Name: tfs.Name, Repeated: tfs.Mode == "REPEATED", Required: tfs.Mode == "REQUIRED", Type: FieldType(tfs.Type), } for _, f := range tfs.Fields { fs.Schema = append(fs.Schema, convertTableFieldSchema(f)) } return fs } func convertTableSchema(ts *bq.TableSchema) Schema { var s Schema for _, f := range ts.Fields { s = append(s, convertTableFieldSchema(f)) } return s } type FieldType string const ( StringFieldType FieldType = "STRING" IntegerFieldType FieldType = "INTEGER" FloatFieldType FieldType = "FLOAT" BooleanFieldType FieldType = "BOOLEAN" TimestampFieldType FieldType = "TIMESTAMP" RecordFieldType FieldType = "RECORD" ) var errNoStruct = errors.New("bigquery: can only infer schema from struct or pointer to struct") var errUnsupportedFieldType = errors.New("bigquery: unsupported type of field in struct") // InferSchema tries to derive a BigQuery schema from the supplied struct value. // NOTE: All fields in the returned Schema are configured to be required, // unless the corresponding field in the supplied struct is a slice or array. // It is considered an error if the struct (including nested structs) contains // any exported fields that are pointers or one of the following types: // map, interface, complex64, complex128, func, chan. // In these cases, an error will be returned. // Future versions may handle these cases without error. func InferSchema(st interface{}) (Schema, error) { return inferStruct(reflect.TypeOf(st)) } func inferStruct(rt reflect.Type) (Schema, error) { switch rt.Kind() { case reflect.Struct: return inferFields(rt) default: return nil, errNoStruct } } // inferFieldSchema infers the FieldSchema for a Go type func inferFieldSchema(rt reflect.Type) (*FieldSchema, error) { switch { case isByteSlice(rt): return &FieldSchema{Required: true, Type: StringFieldType}, nil case isTimeTime(rt): return &FieldSchema{Required: true, Type: TimestampFieldType}, nil case isRepeated(rt): et := rt.Elem() if isRepeated(et) && !isByteSlice(et) { // Multi dimensional slices/arrays are not supported by BigQuery return nil, errUnsupportedFieldType } f, err := inferFieldSchema(et) if err != nil { return nil, err } f.Repeated = true f.Required = false return f, nil case isStruct(rt): nested, err := inferFields(rt) if err != nil { return nil, err } return &FieldSchema{Required: true, Type: RecordFieldType, Schema: nested}, nil } switch rt.Kind() { case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr: return &FieldSchema{Required: true, Type: IntegerFieldType}, nil case reflect.String: return &FieldSchema{Required: true, Type: StringFieldType}, nil case reflect.Bool: return &FieldSchema{Required: true, Type: BooleanFieldType}, nil case reflect.Float32, reflect.Float64: return &FieldSchema{Required: true, Type: FloatFieldType}, nil default: return nil, errUnsupportedFieldType } } // inferFields extracts all exported field types from struct type. func inferFields(rt reflect.Type) (Schema, error) { var s Schema for i := 0; i < rt.NumField(); i++ { field := rt.Field(i) if field.PkgPath != "" { // field is unexported. continue } if field.Anonymous { // TODO(nightlyone) support embedded (see https://github.com/GoogleCloudPlatform/google-cloud-go/issues/238) return nil, errUnsupportedFieldType } f, err := inferFieldSchema(field.Type) if err != nil { return nil, err } f.Name = field.Name s = append(s, f) } return s, nil } func isByteSlice(rt reflect.Type) bool { return rt.Kind() == reflect.Slice && rt.Elem().Kind() == reflect.Uint8 } func isTimeTime(rt reflect.Type) bool { return rt.PkgPath() == "time" && rt.Name() == "Time" } func isStruct(rt reflect.Type) bool { return rt.Kind() == reflect.Struct } func isRepeated(rt reflect.Type) bool { switch rt.Kind() { case reflect.Slice, reflect.Array: return true default: return false } }