contrib: Add vendors to analyze-local-images

pull/104/head
Quentin Machu 8 years ago
parent 80ddc7f949
commit f024576223

@ -0,0 +1,120 @@
{
"ImportPath": "github.com/coreos/clair/contrib/analyze-local-images",
"GoVersion": "go1.5",
"Packages": [
"github.com/coreos/clair/contrib/analyze-local-images"
],
"Deps": [
{
"ImportPath": "github.com/beorn7/perks/quantile",
"Rev": "b965b613227fddccbfffe13eae360ed3fa822f8d"
},
{
"ImportPath": "github.com/coreos/clair/api/context",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/clair/api/v1",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/clair/config",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/clair/database",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/clair/utils",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/clair/worker",
"Comment": "v1.0.0-rc1-40-g20b665f",
"Rev": "20b665f3927ab6627eda2d9450610d589e35b19f"
},
{
"ImportPath": "github.com/coreos/go-systemd/journal",
"Comment": "v4-34-g4f14f6d",
"Rev": "4f14f6deef2da87e4aa59e6c1c1f3e02ba44c5e1"
},
{
"ImportPath": "github.com/coreos/pkg/capnslog",
"Rev": "2c77715c4df99b5420ffcae14ead08f52104065d"
},
{
"ImportPath": "github.com/fatih/color",
"Comment": "v0.1-17-g533cd7f",
"Rev": "533cd7fd8a85905f67a1753afb4deddc85ea174f"
},
{
"ImportPath": "github.com/fernet/fernet-go",
"Rev": "1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2"
},
{
"ImportPath": "github.com/golang/protobuf/proto",
"Rev": "5fc2294e655b78ed8a02082d37808d46c17d7e64"
},
{
"ImportPath": "github.com/julienschmidt/httprouter",
"Comment": "v1.1-14-g21439ef",
"Rev": "21439ef4d70ba4f3e2a5ed9249e7b03af4019b40"
},
{
"ImportPath": "github.com/kr/text",
"Rev": "bb797dc4fb8320488f47bf11de07a733d7233e1f"
},
{
"ImportPath": "github.com/mattn/go-colorable",
"Rev": "9cbef7c35391cca05f15f8181dc0b18bc9736dbb"
},
{
"ImportPath": "github.com/mattn/go-isatty",
"Rev": "56b76bdf51f7708750eac80fa38b952bb9f32639"
},
{
"ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil",
"Rev": "d0c3fe89de86839aecf2e0579c40ba3bb336a453"
},
{
"ImportPath": "github.com/prometheus/client_golang/prometheus",
"Comment": "0.7.0-68-g67994f1",
"Rev": "67994f177195311c3ea3d4407ed0175e34a4256f"
},
{
"ImportPath": "github.com/prometheus/client_model/go",
"Comment": "model-0.0.2-12-gfa8ad6f",
"Rev": "fa8ad6fec33561be4280a8f0514318c79d7f6cb6"
},
{
"ImportPath": "github.com/prometheus/common/expfmt",
"Rev": "dba5e39d4516169e840def50e507ef5f21b985f9"
},
{
"ImportPath": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg",
"Rev": "dba5e39d4516169e840def50e507ef5f21b985f9"
},
{
"ImportPath": "github.com/prometheus/common/model",
"Rev": "dba5e39d4516169e840def50e507ef5f21b985f9"
},
{
"ImportPath": "github.com/prometheus/procfs",
"Rev": "406e5b7bfd8201a36e2bb5f7bdae0b03380c2ce8"
},
{
"ImportPath": "golang.org/x/sys/unix",
"Rev": "eb2c74142fd19a79b3f237334c7384d5167b1b46"
},
{
"ImportPath": "gopkg.in/yaml.v2",
"Rev": "f7716cbe52baa25d2e9b0d0da546fcf909fc16b4"
}
]
}

@ -0,0 +1,5 @@
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.

File diff suppressed because it is too large Load Diff

@ -0,0 +1,292 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile
import (
"math"
"sort"
)
// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}
// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample
func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type invariant func(s *stream, r float64) float64
// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}
// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}
// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targets map[float64]float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for quantile, epsilon := range targets {
if quantile*s.n <= r {
f = (2 * epsilon * r) / quantile
} else {
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}
func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}
// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}
func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}
// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(float64(l) * q)
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}
// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}
// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}
// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}
// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}
func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}
func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}
func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}
type stream struct {
n float64
l []Sample
ƒ invariant
}
func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}
func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}
func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}
func (s *stream) count() int {
return int(s.n)
}
func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}
func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width
for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}
func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@ -0,0 +1,5 @@
CoreOS Project
Copyright 2015 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).

@ -0,0 +1,64 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package context
import (
"net/http"
"strconv"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "api")
promResponseDurationMilliseconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "clair_api_response_duration_milliseconds",
Help: "The duration of time it takes to receieve and write a response to an API request",
Buckets: prometheus.ExponentialBuckets(9.375, 2, 10),
}, []string{"route", "code"})
)
func init() {
prometheus.MustRegister(promResponseDurationMilliseconds)
}
type Handler func(http.ResponseWriter, *http.Request, httprouter.Params, *RouteContext) (route string, status int)
func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
start := time.Now()
route, status := handler(w, r, p, ctx)
statusStr := strconv.Itoa(status)
if status == 0 {
statusStr = "???"
}
utils.PrometheusObserveTimeMilliseconds(promResponseDurationMilliseconds.WithLabelValues(route, statusStr), start)
log.Infof("%s \"%s %s\" %s (%s)", r.RemoteAddr, r.Method, r.RequestURI, statusStr, time.Since(start))
}
}
type RouteContext struct {
Store database.Datastore
Config *config.APIConfig
}

@ -0,0 +1,633 @@
# Clair v1 API
- [Error Handling](#error-handling)
- [Layers](#layers)
- [POST](#post-layers)
- [GET](#get-layersname)
- [DELETE](#delete-layersname)
- [Namespaces](#namespaces)
- [GET](#get-namespaces)
- [Vulnerabilities](#vulnerabilities)
- [List](#get-namespacesnsnamevulnerabilities)
- [POST](#post-namespacesnamevulnerabilities)
- [GET](#get-namespacesnsnamevulnerabilitiesvulnname)
- [PUT](#put-namespacesnsnamevulnerabilitiesvulnname)
- [DELETE](#delete-namespacesnsnamevulnerabilitiesvulnname)
- [Fixes](#fixes)
- [GET](#get-namespacesnsnamevulnerabilitiesvulnnamefixes)
- [PUT](#put-namespacesnsnamevulnerabilitiesvulnnamefixesfeaturename)
- [DELETE](#delete-namespacesnsnamevulnerabilitiesvulnnamefixesfeaturename)
- [Notifications](#notifications)
- [GET](#get-notificationsname)
- [DELETE](#delete-notificationname)
## Error Handling
###### Description
Every route can optionally provide an `Error` property on the response object.
The HTTP status code of the response should indicate what type of failure occurred and how the client should reaction.
###### Client Retry Behavior
| Code | Name | Retry Behavior |
|------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
| 400 | Bad Request | The body of the request invalid. The request either must be changed before being retried or depends on another request being processed before it. |
| 404 | Not Found | The requested resource could not be found. The request must be changed before being retried. |
| 422 | Unprocessable Entity | The request body is valid, but unsupported. This request should never be retried. |
| 500 | Internal Server Error | The server encountered an error while processing the request. This request should be retried without change. |
###### Example Response
```json
HTTP/1.1 400 Bad Request
Content-Type: application/json;charset=utf-8
Server: clair
{
"Error": {
"Message": "example error message"
}
}
```
## Layers
#### POST /layers
###### Description
The POST route for the Layers resource performs the indexing of a Layer from the provided path and displays the provided Layer with an updated `IndexByVersion` property.
This request blocks for the entire duration of the downloading and indexing of the layer.
###### Example Request
```json
POST http://localhost:6060/v1/layers HTTP/1.1
{
"Layer": {
"Name": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
"Path": "/mnt/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
"ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
"Format": "Docker"
}
}
```
###### Example Response
```json
HTTP/1.1 201 Created
Content-Type: application/json;charset=utf-8
Server: clair
{
"Layer": {
"Name": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
"Path": "/mnt/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
"ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
"Format": "Docker",
"IndexedByVersion": 1
}
}
```
#### GET /layers/`:name`
###### Description
The GET route for the Layers resource displays a Layer and optionally all of its features and vulnerabilities.
###### Query Parameters
| Name | Type | Required | Description |
|-----------------|------|----------|-------------------------------------------------------------------------------|
| features | bool | optional | Displays the list of features indexed in this layer and all of its parents. |
| vulnerabilities | bool | optional | Displays the list of vulnerabilities along with the features described above. |
###### Example Request
```
GET http://localhost:6060/v1/layers/17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52?features&vulnerabilities HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Layer": {
"Name": "17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52",
"NamespaceName": "debian:8",
"ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
"IndexedByVersion": 1,
"Features": [
{
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "8.23-4",
"Vulnerabilities": [
{
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Severity": "Low",
"FixedBy": "9.23-5"
}
]
}
]
}
}
```
#### DELETE /layers/`:name`
###### Description
The DELETE route for the Layers resource removes a Layer and all of its children from the database.
###### Example Request
```json
DELETE http://localhost:6060/v1/layers/17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52 HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
```
## Namespaces
#### GET /namespaces
###### Description
The GET route for the Namespaces resource displays a list of namespaces currently being managed.
###### Example Request
```json
GET http://localhost:6060/v1/namespaces HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Namespaces": [
{ "Name": "debian:8" },
{ "Name": "debian:9" }
]
}
```
## Vulnerabilities
#### GET /namespaces/`:nsName`/vulnerabilities
###### Description
The GET route for the Vulnerabilities resource displays the vulnerabilities data for a given namespace.
###### Query Parameters
| Name | Type | Required | Description |
|---------|------|----------|------------------------------------------------------------|
| limit | int | required | Limits the amount of the vunlerabilities data for a given namespace. |
| page | int | required | Displays the specific page of the vunlerabilities data for a given namespace. |
###### Example Request
```json
GET http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities?limit=2 HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Vulnerabilities": [
{
"Name": "CVE-1999-1332",
"NamespaceName": "debian:8",
"Description": "gzexe in the gzip package on Red Hat Linux 5.0 and earlier allows local users to overwrite files of other users via a symlink attack on a temporary file.",
"Link": "https://security-tracker.debian.org/tracker/CVE-1999-1332",
"Severity": "Low"
},
{
"Name": "CVE-1999-1572",
"NamespaceName": "debian:8",
"Description": "cpio on FreeBSD 2.1.0, Debian GNU/Linux 3.0, and possibly other operating systems, uses a 0 umask when creating files using the -O (archive) or -F options, which creates the files with mode 0666 and allows local users to read or overwrite those files.",
"Link": "https://security-tracker.debian.org/tracker/CVE-1999-1572",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 2.1,
"Vectors": "AV:L/AC:L/Au:N/C:P/I:N"
}
}
}
}
],
"NextPage":"gAAAAABW1ABiOlm6KMDKYFE022bEy_IFJdm4ExxTNuJZMN0Eycn0Sut2tOH9bDB4EWGy5s6xwATUHiG-6JXXaU5U32sBs6_DmA=="
}
```
#### POST /namespaces/`:name`/vulnerabilities
###### Description
The POST route for the Vulnerabilities resource creates a new Vulnerability.
###### Example Request
```json
POST http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities HTTP/1.1
{
"Vulnerability": {
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P"
}
}
},
"FixedIn": [
{
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "8.23-1"
}
]
}
}
```
###### Example Response
```json
HTTP/1.1 201 Created
Content-Type: application/json;charset=utf-8
Server: clair
{
"Vulnerability": {
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P"
}
}
},
"FixedIn": [
{
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "8.23-1"
}
]
}
}
```
#### GET /namespaces/`:nsName`/vulnerabilities/`:vulnName`
###### Description
The GET route for the Vulnerabilities resource displays the current data for a given vulnerability and optionally the features that fix it.
###### Query Parameters
| Name | Type | Required | Description |
|---------|------|----------|------------------------------------------------------------|
| fixedIn | bool | optional | Displays the list of features that fix this vulnerability. |
###### Example Request
```json
GET http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471?fixedIn HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Vulnerability": {
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P"
}
}
},
"FixedIn": [
{
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "8.23-1"
}
]
}
}
```
#### PUT /namespaces/`:nsName`/vulnerabilities/`:vulnName`
###### Description
The PUT route for the Vulnerabilities resource updates a given Vulnerability.
The "FixedIn" property of the Vulnerability must be empty or missing.
Fixes should be managed by the Fixes resource.
If this vulnerability was inserted by a Fetcher, changes may be lost when the Fetcher updates.
###### Example Request
```json
PUT http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471
{
"Vulnerability": {
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P"
}
}
}
}
}
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
{
"Vulnerability": {
"Name": "CVE-2014-9471",
"NamespaceName": "debian:8",
"Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
"Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
"Severity": "Low",
"Metadata": {
"NVD": {
"CVSSv2": {
"Score": 7.5,
"Vectors": "AV:N/AC:L/Au:N/C:P/I:P"
}
}
}
}
}
```
#### DELETE /namespaces/`:nsName`/vulnerabilities/`:vulnName`
###### Description
The DELETE route for the Vulnerabilities resource deletes a given Vulnerability.
If this vulnerability was inserted by a Fetcher, it may be re-inserted when the Fetcher updates.
###### Example Request
```json
GET http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471 HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
```
## Fixes
#### GET /namespaces/`:nsName`/vulnerabilities/`:vulnName`/fixes
###### Description
The GET route for the Fixes resource displays the list of Features that fix the given Vulnerability.
###### Example Request
```json
GET http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471/fixes HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Features": [
{
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "8.23-1"
}
]
}
```
#### PUT /namespaces/`:nsName`/vulnerabilities/`:vulnName`/fixes/`:featureName`
###### Description
The PUT route for the Fixes resource updates a Feature that is the fix for a given Vulnerability.
###### Example Request
```json
PUT http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471/fixes/coreutils HTTP/1.1
{
"Feature": {
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "4.24-9"
}
}
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
{
"Feature": {
"Name": "coreutils",
"NamespaceName": "debian:8",
"Version": "4.24-9"
}
}
```
#### DELETE /namespaces/`:nsName`/vulnerabilities/`:vulnName`/fixes/`:featureName`
###### Description
The DELETE route for the Fixes resource removes a Feature as fix for the given Vulnerability.
###### Example Request
```json
DELETE http://localhost:6060/v1/namespaces/debian%3A8/vulnerabilities/CVE-2014-9471/fixes/coreutils
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
```
## Notifications
#### GET /notifications/`:name`
###### Description
The GET route for the Notifications resource displays a notification that a Vulnerability has been updated.
This route supports simultaneous pagination for both the `Old` and `New` Vulnerabilities' `LayersIntroducingVulnerability` property which can be extremely long.
###### Query Parameters
| Name | Type | Required | Description |
|-------|--------|----------|---------------------------------------------------------------------------------------------------------------|
| page | string | optional | Displays the specific page of the "LayersIntroducingVulnerability" property on New and Old vulnerabilities. |
| limit | int | optional | Limits the amount of results in the "LayersIntroducingVulnerability" property on New and Old vulnerabilities. |
###### Example Request
```json
GET http://localhost:6060/v1/notifications/ec45ec87-bfc8-4129-a1c3-d2b82622175a?limit=2 HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Content-Type: application/json;charset=utf-8
Server: clair
{
"Notification": {
"Name": "ec45ec87-bfc8-4129-a1c3-d2b82622175a",
"Created": "1456247389",
"Notified": "1456246708",
"Limit": 2,
"Page": "gAAAAABWzJaC2JCH6Apr_R1f2EkjGdibnrKOobTcYXBWl6t0Cw6Q04ENGIymB6XlZ3Zi0bYt2c-2cXe43fvsJ7ECZhZz4P8C8F9efr_SR0HPiejzQTuG0qAzeO8klogFfFjSz2peBvgP",
"NextPage": "gAAAAABWzJaCTyr6QXP2aYsCwEZfWIkU2GkNplSMlTOhLJfiR3LorBv8QYgEIgyOvZRmHQEzJKvkI6TP2PkRczBkcD17GE89btaaKMqEX14yHDgyfQvdasW1tj3-5bBRt0esKi9ym5En",
"New": {
"Vulnerability": {
"Name": "CVE-TEST",
"NamespaceName": "debian:8",
"Description": "New CVE",
"Severity": "Low",
"FixedIn": [
{
"Name": "grep",
"NamespaceName": "debian:8",
"Version": "2.25"
}
]
},
"LayersIntroducingVulnerability": [
"3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d3f4be1916b12d.9673fdf7-b81a-4b3e-acf8-e551ef155449",
"523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6"
]
},
"Old": {
"Vulnerability": {
"Name": "CVE-TEST",
"NamespaceName": "debian:8",
"Description": "New CVE",
"Severity": "Low",
"FixedIn": []
},
"LayersIntroducingVulnerability": [
"3b59c795b34670618fbcace4dac7a27c5ecec156812c9e2c90d3f4be1916b12d.9673fdf7-b81a-4b3e-acf8-e551ef155449",
"523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6"
]
}
}
}
```
#### DELETE /notifications/`:name`
###### Description
The delete route for the Notifications resource marks a Notification as read.
If a notification is not marked as read, Clair will continue to notify the provided endpoints.
The time at which this Notification was marked as read can be seen in the `Notified` property of the response GET route for Notification.
###### Example Request
```json
DELETE http://localhost:6060/v1/notification/ec45ec87-bfc8-4129-a1c3-d2b82622175a HTTP/1.1
```
###### Example Response
```json
HTTP/1.1 200 OK
Server: clair
```

@ -0,0 +1,318 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
"github.com/fernet/fernet-go"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "v1")
type Error struct {
Message string `json:"Layer`
}
type Layer struct {
Name string `json:"Name,omitempty"`
NamespaceName string `json:"NamespaceName,omitempty"`
Path string `json:"Path,omitempty"`
ParentName string `json:"ParentName,omitempty"`
Format string `json:"Format,omitempty"`
IndexedByVersion int `json:"IndexedByVersion,omitempty"`
Features []Feature `json:"Features,omitempty"`
}
func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer {
layer := Layer{
Name: dbLayer.Name,
IndexedByVersion: dbLayer.EngineVersion,
}
if dbLayer.Parent != nil {
layer.ParentName = dbLayer.Parent.Name
}
if dbLayer.Namespace != nil {
layer.NamespaceName = dbLayer.Namespace.Name
}
if withFeatures || withVulnerabilities && dbLayer.Features != nil {
for _, dbFeatureVersion := range dbLayer.Features {
feature := Feature{
Name: dbFeatureVersion.Feature.Name,
NamespaceName: dbFeatureVersion.Feature.Namespace.Name,
Version: dbFeatureVersion.Version.String(),
AddedBy: dbFeatureVersion.AddedBy.Name,
}
for _, dbVuln := range dbFeatureVersion.AffectedBy {
vuln := Vulnerability{
Name: dbVuln.Name,
NamespaceName: dbVuln.Namespace.Name,
Description: dbVuln.Description,
Link: dbVuln.Link,
Severity: string(dbVuln.Severity),
Metadata: dbVuln.Metadata,
}
if dbVuln.FixedBy != types.MaxVersion {
vuln.FixedBy = dbVuln.FixedBy.String()
}
feature.Vulnerabilities = append(feature.Vulnerabilities, vuln)
}
layer.Features = append(layer.Features, feature)
}
}
return layer
}
type Namespace struct {
Name string `json:"Name,omitempty"`
}
type Vulnerability struct {
Name string `json:"Name,omitempty"`
NamespaceName string `json:"NamespaceName,omitempty"`
Description string `json:"Description,omitempty"`
Link string `json:"Link,omitempty"`
Severity string `json:"Severity,omitempty"`
Metadata map[string]interface{} `json:"Metadata,omitempty"`
FixedBy string `json:"FixedBy,omitempty"`
FixedIn []Feature `json:"FixedIn,omitempty"`
}
func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
severity := types.Priority(v.Severity)
if !severity.IsValid() {
return database.Vulnerability{}, errors.New("Invalid severity")
}
var dbFeatures []database.FeatureVersion
for _, feature := range v.FixedIn {
dbFeature, err := feature.DatabaseModel()
if err != nil {
return database.Vulnerability{}, err
}
dbFeatures = append(dbFeatures, dbFeature)
}
return database.Vulnerability{
Name: v.Name,
Namespace: database.Namespace{Name: v.NamespaceName},
Description: v.Description,
Link: v.Link,
Severity: severity,
Metadata: v.Metadata,
FixedIn: dbFeatures,
}, nil
}
func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn bool) Vulnerability {
vuln := Vulnerability{
Name: dbVuln.Name,
NamespaceName: dbVuln.Namespace.Name,
Description: dbVuln.Description,
Link: dbVuln.Link,
Severity: string(dbVuln.Severity),
Metadata: dbVuln.Metadata,
}
if withFixedIn {
for _, dbFeatureVersion := range dbVuln.FixedIn {
vuln.FixedIn = append(vuln.FixedIn, FeatureFromDatabaseModel(dbFeatureVersion))
}
}
return vuln
}
type Feature struct {
Name string `json:"Name,omitempty"`
NamespaceName string `json:"NamespaceName,omitempty"`
Version string `json:"Version,omitempty"`
Vulnerabilities []Vulnerability `json:"Vulnerabilities,omitempty"`
AddedBy string `json:"AddedBy,omitempty"`
}
func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature {
versionStr := dbFeatureVersion.Version.String()
if versionStr == types.MaxVersion.String() {
versionStr = "None"
}
return Feature{
Name: dbFeatureVersion.Feature.Name,
NamespaceName: dbFeatureVersion.Feature.Namespace.Name,
Version: versionStr,
AddedBy: dbFeatureVersion.AddedBy.Name,
}
}
func (f Feature) DatabaseModel() (database.FeatureVersion, error) {
var version types.Version
if f.Version == "None" {
version = types.MaxVersion
} else {
var err error
version, err = types.NewVersion(f.Version)
if err != nil {
return database.FeatureVersion{}, err
}
}
return database.FeatureVersion{
Feature: database.Feature{
Name: f.Name,
Namespace: database.Namespace{Name: f.NamespaceName},
},
Version: version,
}, nil
}
type Notification struct {
Name string `json:"Name,omitempty"`
Created string `json:"Created,omitempty"`
Notified string `json:"Notified,omitempty"`
Deleted string `json:"Deleted,omitempty"`
Limit int `json:"Limit,omitempty"`
Page string `json:"Page,omitempty"`
NextPage string `json:"NextPage,omitempty"`
Old *VulnerabilityWithLayers `json:"Old,omitempty"`
New *VulnerabilityWithLayers `json:"New,omitempty"`
}
func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotification, limit int, pageToken string, nextPage database.VulnerabilityNotificationPageNumber, key string) Notification {
var oldVuln *VulnerabilityWithLayers
if dbNotification.OldVulnerability != nil {
v := VulnerabilityWithLayersFromDatabaseModel(*dbNotification.OldVulnerability)
oldVuln = &v
}
var newVuln *VulnerabilityWithLayers
if dbNotification.NewVulnerability != nil {
v := VulnerabilityWithLayersFromDatabaseModel(*dbNotification.NewVulnerability)
newVuln = &v
}
var nextPageStr string
if nextPage != database.NoVulnerabilityNotificationPage {
nextPageBytes, _ := tokenMarshal(nextPage, key)
nextPageStr = string(nextPageBytes)
}
var created, notified, deleted string
if !dbNotification.Created.IsZero() {
created = fmt.Sprintf("%d", dbNotification.Created.Unix())
}
if !dbNotification.Notified.IsZero() {
notified = fmt.Sprintf("%d", dbNotification.Notified.Unix())
}
if !dbNotification.Deleted.IsZero() {
deleted = fmt.Sprintf("%d", dbNotification.Deleted.Unix())
}
// TODO(jzelinskie): implement "changed" key
fmt.Println(dbNotification.Deleted.IsZero())
return Notification{
Name: dbNotification.Name,
Created: created,
Notified: notified,
Deleted: deleted,
Limit: limit,
Page: pageToken,
NextPage: nextPageStr,
Old: oldVuln,
New: newVuln,
}
}
type VulnerabilityWithLayers struct {
Vulnerability *Vulnerability `json:"Vulnerability,omitempty"`
LayersIntroducingVulnerability []string `json:"LayersIntroducingVulnerability,omitempty"`
}
func VulnerabilityWithLayersFromDatabaseModel(dbVuln database.Vulnerability) VulnerabilityWithLayers {
vuln := VulnerabilityFromDatabaseModel(dbVuln, true)
var layers []string
for _, layer := range dbVuln.LayersIntroducingVulnerability {
layers = append(layers, layer.Name)
}
return VulnerabilityWithLayers{
Vulnerability: &vuln,
LayersIntroducingVulnerability: layers,
}
}
type LayerEnvelope struct {
Layer *Layer `json:"Layer,omitempty"`
Error *Error `json:"Error,omitempty"`
}
type NamespaceEnvelope struct {
Namespaces *[]Namespace `json:"Namespaces,omitempty"`
Error *Error `json:"Error,omitempty"`
}
type VulnerabilityEnvelope struct {
Vulnerability *Vulnerability `json:"Vulnerability,omitempty"`
Vulnerabilities *[]Vulnerability `json:"Vulnerabilities,omitempty"`
NextPage string `json:"NextPage,omitempty"`
Error *Error `json:"Error,omitempty"`
}
type NotificationEnvelope struct {
Notification *Notification `json:"Notification,omitempty"`
Error *Error `json:"Error,omitempty"`
}
type FeatureEnvelope struct {
Feature *Feature `json:"Feature,omitempty"`
Features *[]Feature `json:"Features,omitempty"`
Error *Error `json:"Error,omitempty"`
}
func tokenUnmarshal(token string, key string, v interface{}) error {
k, _ := fernet.DecodeKey(key)
msg := fernet.VerifyAndDecrypt([]byte(token), time.Hour, []*fernet.Key{k})
if msg == nil {
return errors.New("invalid or expired pagination token")
}
return json.NewDecoder(bytes.NewBuffer(msg)).Decode(&v)
}
func tokenMarshal(v interface{}, key string) ([]byte, error) {
var buf bytes.Buffer
err := json.NewEncoder(&buf).Encode(v)
if err != nil {
return nil, err
}
k, _ := fernet.DecodeKey(key)
return fernet.EncryptAndSign(buf.Bytes(), k)
}

@ -0,0 +1,56 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package v1 implements the first version of the Clair API.
package v1
import (
"github.com/julienschmidt/httprouter"
"github.com/coreos/clair/api/context"
)
// NewRouter creates an HTTP router for version 1 of the Clair API.
func NewRouter(ctx *context.RouteContext) *httprouter.Router {
router := httprouter.New()
// Layers
router.POST("/layers", context.HTTPHandler(postLayer, ctx))
router.GET("/layers/:layerName", context.HTTPHandler(getLayer, ctx))
router.DELETE("/layers/:layerName", context.HTTPHandler(deleteLayer, ctx))
// Namespaces
router.GET("/namespaces", context.HTTPHandler(getNamespaces, ctx))
// Vulnerabilities
router.GET("/namespaces/:namespaceName/vulnerabilities", context.HTTPHandler(getVulnerabilities, ctx))
router.POST("/namespaces/:namespaceName/vulnerabilities", context.HTTPHandler(postVulnerability, ctx))
router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(getVulnerability, ctx))
router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(putVulnerability, ctx))
router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(deleteVulnerability, ctx))
// Fixes
router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes", context.HTTPHandler(getFixes, ctx))
router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", context.HTTPHandler(putFix, ctx))
router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", context.HTTPHandler(deleteFix, ctx))
// Notifications
router.GET("/notifications/:notificationName", context.HTTPHandler(getNotification, ctx))
router.DELETE("/notifications/:notificationName", context.HTTPHandler(deleteNotification, ctx))
// Metrics
router.GET("/metrics", context.HTTPHandler(getMetrics, ctx))
return router
}

@ -0,0 +1,498 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"compress/gzip"
"encoding/json"
"io"
"net/http"
"strconv"
"strings"
"github.com/julienschmidt/httprouter"
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/api/context"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker"
)
const (
// These are the route identifiers for prometheus.
postLayerRoute = "v1/postLayer"
getLayerRoute = "v1/getLayer"
deleteLayerRoute = "v1/deleteLayer"
getNamespacesRoute = "v1/getNamespaces"
getVulnerabilitiesRoute = "v1/getVulnerabilities"
postVulnerabilityRoute = "v1/postVulnerability"
getVulnerabilityRoute = "v1/getVulnerability"
putVulnerabilityRoute = "v1/putVulnerability"
deleteVulnerabilityRoute = "v1/deleteVulnerability"
getFixesRoute = "v1/getFixes"
putFixRoute = "v1/putFix"
deleteFixRoute = "v1/deleteFix"
getNotificationRoute = "v1/getNotification"
deleteNotificationRoute = "v1/deleteNotification"
getMetricsRoute = "v1/getMetrics"
// maxBodySize restricts client request bodies to 1MiB.
maxBodySize int64 = 1048576
// statusUnprocessableEntity represents the 422 (Unprocessable Entity) status code, which means
// the server understands the content type of the request entity
// (hence a 415(Unsupported Media Type) status code is inappropriate), and the syntax of the
// request entity is correct (thus a 400 (Bad Request) status code is inappropriate) but was
// unable to process the contained instructions.
statusUnprocessableEntity = 422
)
func decodeJSON(r *http.Request, v interface{}) error {
defer r.Body.Close()
return json.NewDecoder(io.LimitReader(r.Body, maxBodySize)).Decode(v)
}
func writeResponse(w http.ResponseWriter, r *http.Request, status int, resp interface{}) {
// Headers must be written before the response.
header := w.Header()
header.Set("Content-Type", "application/json;charset=utf-8")
header.Set("Server", "clair")
// Gzip the response if the client supports it.
var writer io.Writer = w
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
gzipWriter := gzip.NewWriter(w)
defer gzipWriter.Close()
writer = gzipWriter
header.Set("Content-Encoding", "gzip")
}
// Write the response.
w.WriteHeader(status)
err := json.NewEncoder(writer).Encode(resp)
if err != nil {
switch err.(type) {
case *json.MarshalerError, *json.UnsupportedTypeError, *json.UnsupportedValueError:
panic("v1: failed to marshal response: " + err.Error())
default:
log.Warningf("failed to write response: %s", err.Error())
}
}
}
func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
request := LayerEnvelope{}
err := decodeJSON(r, &request)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, LayerEnvelope{Error: &Error{err.Error()}})
return postLayerRoute, http.StatusBadRequest
}
if request.Layer == nil {
writeResponse(w, r, http.StatusBadRequest, LayerEnvelope{Error: &Error{"failed to provide layer"}})
return postLayerRoute, http.StatusBadRequest
}
err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Format)
if err != nil {
if err == utils.ErrCouldNotExtract ||
err == utils.ErrExtractedFileTooBig ||
err == worker.ErrUnsupported {
writeResponse(w, r, statusUnprocessableEntity, LayerEnvelope{Error: &Error{err.Error()}})
return postLayerRoute, statusUnprocessableEntity
}
if _, badreq := err.(*cerrors.ErrBadRequest); badreq {
writeResponse(w, r, http.StatusBadRequest, LayerEnvelope{Error: &Error{err.Error()}})
return postLayerRoute, http.StatusBadRequest
}
writeResponse(w, r, http.StatusInternalServerError, LayerEnvelope{Error: &Error{err.Error()}})
return postLayerRoute, http.StatusInternalServerError
}
writeResponse(w, r, http.StatusCreated, LayerEnvelope{Layer: &Layer{
Name: request.Layer.Name,
ParentName: request.Layer.ParentName,
Path: request.Layer.Path,
Format: request.Layer.Format,
IndexedByVersion: worker.Version,
}})
return postLayerRoute, http.StatusCreated
}
func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
_, withFeatures := r.URL.Query()["features"]
_, withVulnerabilities := r.URL.Query()["vulnerabilities"]
dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
return getLayerRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, LayerEnvelope{Error: &Error{err.Error()}})
return getLayerRoute, http.StatusInternalServerError
}
layer := LayerFromDatabaseModel(dbLayer, withFeatures, withVulnerabilities)
writeResponse(w, r, http.StatusOK, LayerEnvelope{Layer: &layer})
return getLayerRoute, http.StatusOK
}
func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteLayer(p.ByName("layerName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
return deleteLayerRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, LayerEnvelope{Error: &Error{err.Error()}})
return deleteLayerRoute, http.StatusInternalServerError
}
w.WriteHeader(http.StatusOK)
return deleteLayerRoute, http.StatusOK
}
func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
dbNamespaces, err := ctx.Store.ListNamespaces()
if err != nil {
writeResponse(w, r, http.StatusInternalServerError, NamespaceEnvelope{Error: &Error{err.Error()}})
return getNamespacesRoute, http.StatusInternalServerError
}
var namespaces []Namespace
for _, dbNamespace := range dbNamespaces {
namespaces = append(namespaces, Namespace{Name: dbNamespace.Name})
}
writeResponse(w, r, http.StatusOK, NamespaceEnvelope{Namespaces: &namespaces})
return getNamespacesRoute, http.StatusOK
}
func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
query := r.URL.Query()
limitStrs, limitExists := query["limit"]
if !limitExists {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"must provide limit query parameter"}})
return getVulnerabilitiesRoute, http.StatusBadRequest
}
limit, err := strconv.Atoi(limitStrs[0])
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"invalid limit format: " + err.Error()}})
return getVulnerabilitiesRoute, http.StatusBadRequest
} else if limit < 0 {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"limit value should not be less than zero"}})
return getVulnerabilitiesRoute, http.StatusBadRequest
}
page := 0
pageStrs, pageExists := query["page"]
if pageExists {
err = tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"invalid page format: " + err.Error()}})
return getNotificationRoute, http.StatusBadRequest
}
}
namespace := p.ByName("namespaceName")
if namespace == "" {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"namespace should not be empty"}})
return getNotificationRoute, http.StatusBadRequest
}
dbVulns, nextPage, err := ctx.Store.ListVulnerabilities(namespace, limit, page)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilityRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilitiesRoute, http.StatusInternalServerError
}
var vulns []Vulnerability
for _, dbVuln := range dbVulns {
vuln := VulnerabilityFromDatabaseModel(dbVuln, false)
vulns = append(vulns, vuln)
}
var nextPageStr string
if nextPage != -1 {
nextPageBytes, err := tokenMarshal(nextPage, ctx.Config.PaginationKey)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"failed to marshal token: " + err.Error()}})
return getNotificationRoute, http.StatusBadRequest
}
nextPageStr = string(nextPageBytes)
}
writeResponse(w, r, http.StatusOK, VulnerabilityEnvelope{Vulnerabilities: &vulns, NextPage: nextPageStr})
return getVulnerabilitiesRoute, http.StatusOK
}
func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
request := VulnerabilityEnvelope{}
err := decodeJSON(r, &request)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return postVulnerabilityRoute, http.StatusBadRequest
}
if request.Vulnerability == nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
return postVulnerabilityRoute, http.StatusBadRequest
}
vuln, err := request.Vulnerability.DatabaseModel()
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return postVulnerabilityRoute, http.StatusBadRequest
}
err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return postVulnerabilityRoute, http.StatusBadRequest
default:
writeResponse(w, r, http.StatusInternalServerError, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return postVulnerabilityRoute, http.StatusInternalServerError
}
}
writeResponse(w, r, http.StatusCreated, VulnerabilityEnvelope{Vulnerability: request.Vulnerability})
return postVulnerabilityRoute, http.StatusCreated
}
func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
_, withFixedIn := r.URL.Query()["fixedIn"]
dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilityRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilityRoute, http.StatusInternalServerError
}
vuln := VulnerabilityFromDatabaseModel(dbVuln, withFixedIn)
writeResponse(w, r, http.StatusOK, VulnerabilityEnvelope{Vulnerability: &vuln})
return getVulnerabilityRoute, http.StatusOK
}
func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
request := VulnerabilityEnvelope{}
err := decodeJSON(r, &request)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return putVulnerabilityRoute, http.StatusBadRequest
}
if request.Vulnerability == nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
return putVulnerabilityRoute, http.StatusBadRequest
}
if len(request.Vulnerability.FixedIn) != 0 {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"Vulnerability.FixedIn must be empty"}})
return putVulnerabilityRoute, http.StatusBadRequest
}
vuln, err := request.Vulnerability.DatabaseModel()
if err != nil {
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return putVulnerabilityRoute, http.StatusBadRequest
}
vuln.Namespace.Name = p.ByName("namespaceName")
vuln.Name = p.ByName("vulnerabilityName")
err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return putVulnerabilityRoute, http.StatusBadRequest
default:
writeResponse(w, r, http.StatusInternalServerError, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return putVulnerabilityRoute, http.StatusInternalServerError
}
}
writeResponse(w, r, http.StatusOK, VulnerabilityEnvelope{Vulnerability: request.Vulnerability})
return putVulnerabilityRoute, http.StatusOK
}
func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return deleteVulnerabilityRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return deleteVulnerabilityRoute, http.StatusInternalServerError
}
w.WriteHeader(http.StatusOK)
return deleteVulnerabilityRoute, http.StatusOK
}
func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
return getFixesRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, FeatureEnvelope{Error: &Error{err.Error()}})
return getFixesRoute, http.StatusInternalServerError
}
vuln := VulnerabilityFromDatabaseModel(dbVuln, true)
writeResponse(w, r, http.StatusOK, FeatureEnvelope{Features: &vuln.FixedIn})
return getFixesRoute, http.StatusOK
}
func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
request := FeatureEnvelope{}
err := decodeJSON(r, &request)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{err.Error()}})
return putFixRoute, http.StatusBadRequest
}
if request.Feature == nil {
writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{"failed to provide feature"}})
return putFixRoute, http.StatusBadRequest
}
if request.Feature.Name != p.ByName("fixName") {
writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{"feature name in URL and JSON do not match"}})
return putFixRoute, http.StatusBadRequest
}
dbFix, err := request.Feature.DatabaseModel()
if err != nil {
writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{err.Error()}})
return putFixRoute, http.StatusBadRequest
}
err = ctx.Store.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []database.FeatureVersion{dbFix})
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{err.Error()}})
return putFixRoute, http.StatusBadRequest
default:
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
return putFixRoute, http.StatusNotFound
}
writeResponse(w, r, http.StatusInternalServerError, FeatureEnvelope{Error: &Error{err.Error()}})
return putFixRoute, http.StatusInternalServerError
}
}
writeResponse(w, r, http.StatusOK, FeatureEnvelope{Feature: request.Feature})
return putFixRoute, http.StatusOK
}
func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
return deleteFixRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, FeatureEnvelope{Error: &Error{err.Error()}})
return deleteFixRoute, http.StatusInternalServerError
}
w.WriteHeader(http.StatusOK)
return deleteFixRoute, http.StatusOK
}
func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
query := r.URL.Query()
limitStrs, limitExists := query["limit"]
if !limitExists {
writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"must provide limit query parameter"}})
return getNotificationRoute, http.StatusBadRequest
}
limit, err := strconv.Atoi(limitStrs[0])
if err != nil {
writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"invalid limit format: " + err.Error()}})
return getNotificationRoute, http.StatusBadRequest
}
var pageToken string
page := database.VulnerabilityNotificationFirstPage
pageStrs, pageExists := query["page"]
if pageExists {
err := tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"invalid page format: " + err.Error()}})
return getNotificationRoute, http.StatusBadRequest
}
pageToken = pageStrs[0]
} else {
pageTokenBytes, err := tokenMarshal(page, ctx.Config.PaginationKey)
if err != nil {
writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"failed to marshal token: " + err.Error()}})
return getNotificationRoute, http.StatusBadRequest
}
pageToken = string(pageTokenBytes)
}
dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
return deleteNotificationRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, NotificationEnvelope{Error: &Error{err.Error()}})
return getNotificationRoute, http.StatusInternalServerError
}
notification := NotificationFromDatabaseModel(dbNotification, limit, pageToken, nextPage, ctx.Config.PaginationKey)
writeResponse(w, r, http.StatusOK, NotificationEnvelope{Notification: &notification})
return getNotificationRoute, http.StatusOK
}
func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteNotification(p.ByName("notificationName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
return deleteNotificationRoute, http.StatusNotFound
} else if err != nil {
writeResponse(w, r, http.StatusInternalServerError, NotificationEnvelope{Error: &Error{err.Error()}})
return deleteNotificationRoute, http.StatusInternalServerError
}
w.WriteHeader(http.StatusOK)
return deleteNotificationRoute, http.StatusOK
}
func getMetrics(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
prometheus.Handler().ServeHTTP(w, r)
return getMetricsRoute, 0
}

@ -0,0 +1,128 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"io/ioutil"
"os"
"time"
"github.com/fernet/fernet-go"
"gopkg.in/yaml.v2"
)
// File represents a YAML configuration file that namespaces all Clair
// configuration under the top-level "clair" key.
type File struct {
Clair Config `yaml:"clair"`
}
// Config is the global configuration for an instance of Clair.
type Config struct {
Database *DatabaseConfig
Updater *UpdaterConfig
Notifier *NotifierConfig
API *APIConfig
}
// DatabaseConfig is the configuration used to specify how Clair connects
// to a database.
type DatabaseConfig struct {
Source string
CacheSize int
}
// UpdaterConfig is the configuration for the Updater service.
type UpdaterConfig struct {
Interval time.Duration
}
// NotifierConfig is the configuration for the Notifier service and its registered notifiers.
type NotifierConfig struct {
Attempts int
RenotifyInterval time.Duration
Params map[string]interface{} `yaml:",inline"`
}
// APIConfig is the configuration for the API service.
type APIConfig struct {
Port int
HealthPort int
Timeout time.Duration
PaginationKey string
CertFile, KeyFile, CAFile string
}
// DefaultConfig is a configuration that can be used as a fallback value.
var DefaultConfig = Config{
Database: &DatabaseConfig{
CacheSize: 16384,
},
Updater: &UpdaterConfig{
Interval: 1 * time.Hour,
},
API: &APIConfig{
Port: 6060,
HealthPort: 6061,
Timeout: 900 * time.Second,
},
Notifier: &NotifierConfig{
Attempts: 5,
RenotifyInterval: 2 * time.Hour,
},
}
// Load is a shortcut to open a file, read it, and generate a Config.
// It supports relative and absolute paths. Given "", it returns DefaultConfig.
func Load(path string) (config *Config, err error) {
config = &DefaultConfig
if path == "" {
return
}
f, err := os.Open(os.ExpandEnv(path))
if err != nil {
return
}
defer f.Close()
d, err := ioutil.ReadAll(f)
if err != nil {
return
}
var cfgFile File
err = yaml.Unmarshal(d, &cfgFile)
if err != nil {
return
}
config = &cfgFile.Clair
// Generate a pagination key if none is provided.
if config.API.PaginationKey == "" {
var key fernet.Key
if err = key.Generate(); err != nil {
return
}
config.API.PaginationKey = key.Encode()
} else {
_, err = fernet.DecodeKey(config.API.PaginationKey)
if err != nil {
return
}
}
return
}

@ -0,0 +1,155 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package database defines the Clair's models and a common interface for database implementations.
package database
import (
"errors"
"time"
)
var (
// ErrBackendException is an error that occurs when the database backend does
// not work properly (ie. unreachable).
ErrBackendException = errors.New("database: an error occured when querying the backend")
// ErrInconsistent is an error that occurs when a database consistency check
// fails (ie. when an entity which is supposed to be unique is detected twice)
ErrInconsistent = errors.New("database: inconsistent database")
// ErrCantOpen is an error that occurs when the database could not be opened
ErrCantOpen = errors.New("database: could not open database")
)
// Datastore is the interface that describes a database backend implementation.
type Datastore interface {
// # Namespace
// ListNamespaces returns the entire list of known Namespaces.
ListNamespaces() ([]Namespace, error)
// # Layer
// InsertLayer stores a Layer in the database.
// A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory.
// If a Parent is specified, it is expected that it has been retrieved using FindLayer.
// If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher
// than the stored one, the stored Layer should be updated.
// The function has to be idempotent, inserting a layer that already exists shouln'd return an
// error.
InsertLayer(Layer) error
// FindLayer retrieves a Layer from the database.
// withFeatures specifies whether the Features field should be filled. When withVulnerabilities is
// true, the Features field should be filled and their AffectedBy fields should contain every
// vulnerabilities that affect them.
FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error)
// DeleteLayer deletes a Layer from the database and every layers that are based on it,
// recursively.
DeleteLayer(name string) error
// # Vulnerability
// ListVulnerabilities returns the list of vulnerabilies of a certain Namespace.
// The Limit and page parameters are used to paginate the return list.
// The first given page should be 0. The function will then return the next available page.
// If there is no more page, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if
// necessary. A vulnerability is uniquely identified by its Namespace and its Name.
// The FixedIn field may only contain a partial list of Features that are affected by the
// Vulnerability, along with the version in which the vulnerability is fixed. It is the
// responsibility of the implementation to update the list properly. A version equals to
// types.MinVersion means that the given Feature is not being affected by the Vulnerability at
// all and thus, should be removed from the list. It is important that Features should be unique
// in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as
// a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and
// Features are Namespaced (i.e. specific to one operating system).
// Each vulnerability insertion or update has to create a Notification that will contain the
// old and the updated Vulnerability, unless createNotification equals to true.
InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list.
FindVulnerability(namespaceName, name string) (Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database.
// It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to
// the specified Vulnerability in the database.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the
// database. It can be used to store the fact that a Vulnerability no longer affects the given
// Feature in any Version.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
// # Notification
// GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a
// Notification that should be handled. The renotify interval defines how much time after being
// marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should
// be returned again by this function. A Notification for which there is a valid Lock with the
// same Name should not be returned.
GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error)
// GetNotification returns a Notification, including its OldVulnerability and NewVulnerability
// fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with
// every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion).
// The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first
// given page should be VulnerabilityNotificationFirstPage. The function will then return the next
// availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned.
GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
// SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for
// GetAvailableNotification, until the renotify duration is elapsed.
SetNotificationNotified(name string) error
// DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for
// GetAvailableNotification.
DeleteNotification(name string) error
// # Key/Value
// InsertKeyValue stores or updates a simple key/value pair in the database.
InsertKeyValue(key, value string) error
// GetKeyValue retrieves a value from the database from the given key.
// It returns an empty string if there is no such key.
GetKeyValue(key string) (string, error)
// # Lock
// Lock creates or renew a Lock in the database with the given name, owner and duration.
// After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus,
// let other users create a Lock with the same name. However, the owner can renew its Lock by
// setting renew to true. Lock should not block, it should instead returns whether the Lock has
// been successfully acquired/renewed. If it's the case, the expiration time of that Lock is
// returned as well.
Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time)
// Unlock releases an existing Lock.
Unlock(name, owner string)
// FindLock returns the owner of a Lock specified by the name, and its experation time if it
// exists.
FindLock(name string) (string, time.Time, error)
// # Miscellaneous
// Ping returns the health status of the database.
Ping() bool
// Close closes the database and free any allocated resource.
Close()
}

@ -0,0 +1,119 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
import (
"database/sql/driver"
"encoding/json"
"time"
"github.com/coreos/clair/utils/types"
)
// ID is only meant to be used by database implementations and should never be used for anything else.
type Model struct {
ID int
}
type Layer struct {
Model
Name string
EngineVersion int
Parent *Layer
Namespace *Namespace
Features []FeatureVersion
}
type Namespace struct {
Model
Name string
}
type Feature struct {
Model
Name string
Namespace Namespace
}
type FeatureVersion struct {
Model
Feature Feature
Version types.Version
AffectedBy []Vulnerability
// For output purposes. Only make sense when the feature version is in the context of an image.
AddedBy Layer
}
type Vulnerability struct {
Model
Name string
Namespace Namespace
Description string
Link string
Severity types.Priority
Metadata MetadataMap
FixedIn []FeatureVersion
LayersIntroducingVulnerability []Layer
// For output purposes. Only make sense when the vulnerability
// is already about a specific Feature/FeatureVersion.
FixedBy types.Version `json:",omitempty"`
}
type MetadataMap map[string]interface{}
func (mm *MetadataMap) Scan(value interface{}) error {
val, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(val, mm)
}
func (mm *MetadataMap) Value() (driver.Value, error) {
json, err := json.Marshal(*mm)
return string(json), err
}
type VulnerabilityNotification struct {
Model
Name string
Created time.Time
Notified time.Time
Deleted time.Time
OldVulnerability *Vulnerability
NewVulnerability *Vulnerability
}
type VulnerabilityNotificationPageNumber struct {
// -1 means that we reached the end already.
OldVulnerability int
NewVulnerability int
}
var VulnerabilityNotificationFirstPage = VulnerabilityNotificationPageNumber{0, 0}
var NoVulnerabilityNotificationPage = VulnerabilityNotificationPageNumber{-1, -1}

@ -0,0 +1,43 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
// DebianReleasesMapping translates Debian code names and class names to version numbers
var DebianReleasesMapping = map[string]string{
// Code names
"squeeze": "6",
"wheezy": "7",
"jessie": "8",
"stretch": "9",
"sid": "unstable",
// Class names
"oldstable": "7",
"stable": "8",
"testing": "9",
"unstable": "unstable",
}
// UbuntuReleasesMapping translates Ubuntu code names to version numbers
var UbuntuReleasesMapping = map[string]string{
"precise": "12.04",
"quantal": "12.10",
"raring": "13.04",
"trusty": "14.04",
"utopic": "14.10",
"vivid": "15.04",
"wily": "15.10",
"xenial": "16.04",
}

@ -0,0 +1,240 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"database/sql"
"time"
"github.com/coreos/clair/database"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
if feature.Name == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature")
}
// Do cache lookup.
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("feature").Inc()
id, found := pgSQL.cache.Get("feature:" + feature.Namespace.Name + ":" + feature.Name)
if found {
promCacheHitsTotal.WithLabelValues("feature").Inc()
return id.(int), nil
}
}
// We do `defer observeQueryTime` here because we don't want to observe cached features.
defer observeQueryTime("insertFeature", "all", time.Now())
// Find or create Namespace.
namespaceID, err := pgSQL.insertNamespace(feature.Namespace)
if err != nil {
return 0, err
}
// Find or create Feature.
var id int
err = pgSQL.QueryRow(soiFeature, feature.Name, namespaceID).Scan(&id)
if err != nil {
return 0, handleError("soiFeature", err)
}
if pgSQL.cache != nil {
pgSQL.cache.Add("feature:"+feature.Namespace.Name+":"+feature.Name, id)
}
return id, nil
}
func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) {
if featureVersion.Version.String() == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
}
// Do cache lookup.
cacheIndex := "featureversion:" + featureVersion.Feature.Namespace.Name + ":" + featureVersion.Feature.Name + ":" + featureVersion.Version.String()
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("featureversion").Inc()
id, found := pgSQL.cache.Get(cacheIndex)
if found {
promCacheHitsTotal.WithLabelValues("featureversion").Inc()
return id.(int), nil
}
}
// We do `defer observeQueryTime` here because we don't want to observe cached featureversions.
defer observeQueryTime("insertFeatureVersion", "all", time.Now())
// Find or create Feature first.
t := time.Now()
featureID, err := pgSQL.insertFeature(featureVersion.Feature)
observeQueryTime("insertFeatureVersion", "insertFeature", t)
if err != nil {
return 0, err
}
featureVersion.Feature.ID = featureID
// Try to find the FeatureVersion.
//
// In a populated database, the likelihood of the FeatureVersion already being there is high.
// If we can find it here, we then avoid using a transaction and locking the database.
err = pgSQL.QueryRow(searchFeatureVersion, featureID, &featureVersion.Version).
Scan(&featureVersion.ID)
if err != nil && err != sql.ErrNoRows {
return 0, handleError("searchFeatureVersion", err)
}
if err == nil {
if pgSQL.cache != nil {
pgSQL.cache.Add(cacheIndex, featureVersion.ID)
}
return featureVersion.ID, nil
}
// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return 0, handleError("insertFeatureVersion.Begin()", err)
}
// Lock Vulnerability_Affects_FeatureVersion exclusively.
// We want to prevent InsertVulnerability to modify it.
promConcurrentLockVAFV.Inc()
defer promConcurrentLockVAFV.Dec()
t = time.Now()
_, err = tx.Exec(lockVulnerabilityAffects)
observeQueryTime("insertFeatureVersion", "lock", t)
if err != nil {
tx.Rollback()
return 0, handleError("insertFeatureVersion.lockVulnerabilityAffects", err)
}
// Find or create FeatureVersion.
var newOrExisting string
t = time.Now()
err = tx.QueryRow(soiFeatureVersion, featureID, &featureVersion.Version).
Scan(&newOrExisting, &featureVersion.ID)
observeQueryTime("insertFeatureVersion", "soiFeatureVersion", t)
if err != nil {
tx.Rollback()
return 0, handleError("soiFeatureVersion", err)
}
if newOrExisting == "exi" {
// That featureVersion already exists, return its id.
tx.Commit()
if pgSQL.cache != nil {
pgSQL.cache.Add(cacheIndex, featureVersion.ID)
}
return featureVersion.ID, nil
}
// Link the new FeatureVersion with every vulnerabilities that affect it, by inserting in
// Vulnerability_Affects_FeatureVersion.
t = time.Now()
err = linkFeatureVersionToVulnerabilities(tx, featureVersion)
observeQueryTime("insertFeatureVersion", "linkFeatureVersionToVulnerabilities", t)
if err != nil {
tx.Rollback()
return 0, err
}
// Commit transaction.
err = tx.Commit()
if err != nil {
return 0, handleError("insertFeatureVersion.Commit()", err)
}
if pgSQL.cache != nil {
pgSQL.cache.Add(cacheIndex, featureVersion.ID)
}
return featureVersion.ID, nil
}
// TODO(Quentin-M): Batch me
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
IDs := make([]int, 0, len(featureVersions))
for i := 0; i < len(featureVersions); i++ {
id, err := pgSQL.insertFeatureVersion(featureVersions[i])
if err != nil {
return IDs, err
}
IDs = append(IDs, id)
}
return IDs, nil
}
type vulnerabilityAffectsFeatureVersion struct {
vulnerabilityID int
fixedInID int
fixedInVersion types.Version
}
func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error {
// Select every vulnerability and the fixed version that affect this Feature.
// TODO(Quentin-M): LIMIT
rows, err := tx.Query(searchVulnerabilityFixedInFeature, featureVersion.Feature.ID)
if err != nil {
return handleError("searchVulnerabilityFixedInFeature", err)
}
defer rows.Close()
var affects []vulnerabilityAffectsFeatureVersion
for rows.Next() {
var affect vulnerabilityAffectsFeatureVersion
err := rows.Scan(&affect.fixedInID, &affect.vulnerabilityID, &affect.fixedInVersion)
if err != nil {
return handleError("searchVulnerabilityFixedInFeature.Scan()", err)
}
if featureVersion.Version.Compare(affect.fixedInVersion) < 0 {
// The version of the FeatureVersion we are inserting is lower than the fixed version on this
// Vulnerability, thus, this FeatureVersion is affected by it.
affects = append(affects, affect)
}
}
if err = rows.Err(); err != nil {
return handleError("searchVulnerabilityFixedInFeature.Rows()", err)
}
rows.Close()
// Insert into Vulnerability_Affects_FeatureVersion.
for _, affect := range affects {
// TODO(Quentin-M): Batch me.
_, err := tx.Exec(insertVulnerabilityAffectsFeatureVersion, affect.vulnerabilityID,
featureVersion.ID, affect.fixedInID)
if err != nil {
return handleError("insertVulnerabilityAffectsFeatureVersion", err)
}
}
return nil
}

@ -0,0 +1,83 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"database/sql"
"time"
cerrors "github.com/coreos/clair/utils/errors"
)
// InsertKeyValue stores (or updates) a single key / value tuple.
func (pgSQL *pgSQL) InsertKeyValue(key, value string) (err error) {
if key == "" || value == "" {
log.Warning("could not insert a flag which has an empty name or value")
return cerrors.NewBadRequestError("could not insert a flag which has an empty name or value")
}
defer observeQueryTime("InsertKeyValue", "all", time.Now())
// Upsert.
//
// Note: UPSERT works only on >= PostgreSQL 9.5 which is not yet supported by AWS RDS.
// The best solution is currently the use of http://dba.stackexchange.com/a/13477
// but the key/value storage doesn't need to be super-efficient and super-safe at the
// moment so we can just use a client-side solution with transactions, based on
// http://postgresql.org/docs/current/static/plpgsql-control-structures.html.
// TODO(Quentin-M): Enable Upsert as soon as 9.5 is stable.
for {
// First, try to update.
r, err := pgSQL.Exec(updateKeyValue, value, key)
if err != nil {
return handleError("updateKeyValue", err)
}
if n, _ := r.RowsAffected(); n > 0 {
// Updated successfully.
return nil
}
// Try to insert the key.
// If someone else inserts the same key concurrently, we could get a unique-key violation error.
_, err = pgSQL.Exec(insertKeyValue, key, value)
if err != nil {
if isErrUniqueViolation(err) {
// Got unique constraint violation, retry.
continue
}
return handleError("insertKeyValue", err)
}
return nil
}
}
// GetValue reads a single key / value tuple and returns an empty string if the key doesn't exist.
func (pgSQL *pgSQL) GetKeyValue(key string) (string, error) {
defer observeQueryTime("GetKeyValue", "all", time.Now())
var value string
err := pgSQL.QueryRow(searchKeyValue, key).Scan(&value)
if err == sql.ErrNoRows {
return "", nil
}
if err != nil {
return "", handleError("searchKeyValue", err)
}
return value, nil
}

@ -0,0 +1,406 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"database/sql"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/guregu/null/zero"
)
func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
subquery := "all"
if withFeatures {
subquery += "/features"
} else if withVulnerabilities {
subquery += "/features+vulnerabilities"
}
defer observeQueryTime("FindLayer", subquery, time.Now())
// Find the layer
var layer database.Layer
var parentID zero.Int
var parentName zero.String
var namespaceID zero.Int
var namespaceName sql.NullString
t := time.Now()
err := pgSQL.QueryRow(searchLayer, name).
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID,
&namespaceName)
observeQueryTime("FindLayer", "searchLayer", t)
if err != nil {
return layer, handleError("searchLayer", err)
}
if !parentID.IsZero() {
layer.Parent = &database.Layer{
Model: database.Model{ID: int(parentID.Int64)},
Name: parentName.String,
}
}
if !namespaceID.IsZero() {
layer.Namespace = &database.Namespace{
Model: database.Model{ID: int(namespaceID.Int64)},
Name: namespaceName.String,
}
}
// Find its features
if withFeatures || withVulnerabilities {
// Create a transaction to disable hash/merge joins as our experiments have shown that
// PostgreSQL 9.4 makes bad planning decisions about:
// - joining the layer tree to feature versions and feature
// - joining the feature versions to affected/fixed feature version and vulnerabilities
// It would for instance do a merge join between affected feature versions (300 rows, estimated
// 3000 rows) and fixed in feature version (100k rows). In this case, it is much more
// preferred to use a nested loop.
tx, err := pgSQL.Begin()
if err != nil {
return layer, handleError("FindLayer.Begin()", err)
}
defer tx.Commit()
_, err = tx.Exec(disableHashJoin)
if err != nil {
log.Warningf("FindLayer: could not disable hash join: %s", err)
}
_, err = tx.Exec(disableMergeJoin)
if err != nil {
log.Warningf("FindLayer: could not disable merge join: %s", err)
}
t = time.Now()
featureVersions, err := getLayerFeatureVersions(tx, layer.ID)
observeQueryTime("FindLayer", "getLayerFeatureVersions", t)
if err != nil {
return layer, err
}
layer.Features = featureVersions
if withVulnerabilities {
// Load the vulnerabilities that affect the FeatureVersions.
t = time.Now()
err := loadAffectedBy(tx, layer.Features)
observeQueryTime("FindLayer", "loadAffectedBy", t)
if err != nil {
return layer, err
}
}
}
return layer, nil
}
// getLayerFeatureVersions returns list of database.FeatureVersion that a database.Layer has.
func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion, error) {
var featureVersions []database.FeatureVersion
// Query.
rows, err := tx.Query(searchLayerFeatureVersion, layerID)
if err != nil {
return featureVersions, handleError("searchLayerFeatureVersion", err)
}
defer rows.Close()
// Scan query.
var modification string
mapFeatureVersions := make(map[int]database.FeatureVersion)
for rows.Next() {
var featureVersion database.FeatureVersion
err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID,
&featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID,
&featureVersion.Feature.Name, &featureVersion.ID, &featureVersion.Version,
&featureVersion.AddedBy.ID, &featureVersion.AddedBy.Name)
if err != nil {
return featureVersions, handleError("searchLayerFeatureVersion.Scan()", err)
}
// Do transitive closure.
switch modification {
case "add":
mapFeatureVersions[featureVersion.ID] = featureVersion
case "del":
delete(mapFeatureVersions, featureVersion.ID)
default:
log.Warningf("unknown Layer_diff_FeatureVersion's modification: %s", modification)
return featureVersions, database.ErrInconsistent
}
}
if err = rows.Err(); err != nil {
return featureVersions, handleError("searchLayerFeatureVersion.Rows()", err)
}
// Build result by converting our map to a slice.
for _, featureVersion := range mapFeatureVersions {
featureVersions = append(featureVersions, featureVersion)
}
return featureVersions, nil
}
// loadAffectedBy returns the list of database.Vulnerability that affect the given
// FeatureVersion.
func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error {
if len(featureVersions) == 0 {
return nil
}
// Construct list of FeatureVersion IDs, we will do a single query
featureVersionIDs := make([]int, 0, len(featureVersions))
for i := 0; i < len(featureVersions); i++ {
featureVersionIDs = append(featureVersionIDs, featureVersions[i].ID)
}
rows, err := tx.Query(searchFeatureVersionVulnerability,
buildInputArray(featureVersionIDs))
if err != nil && err != sql.ErrNoRows {
return handleError("searchFeatureVersionVulnerability", err)
}
defer rows.Close()
vulnerabilities := make(map[int][]database.Vulnerability, len(featureVersions))
var featureversionID int
for rows.Next() {
var vulnerability database.Vulnerability
err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy)
if err != nil {
return handleError("searchFeatureVersionVulnerability.Scan()", err)
}
vulnerabilities[featureversionID] = append(vulnerabilities[featureversionID], vulnerability)
}
if err = rows.Err(); err != nil {
return handleError("searchFeatureVersionVulnerability.Rows()", err)
}
// Assign vulnerabilities to every FeatureVersions
for i := 0; i < len(featureVersions); i++ {
featureVersions[i].AffectedBy = vulnerabilities[featureVersions[i].ID]
}
return nil
}
// Internally, only Feature additions/removals are stored for each layer. If a layer has a parent,
// the Feature list will be compared to the parent's Feature list and the difference will be stored.
// Note that when the Namespace of a layer differs from its parent, it is expected that several
// Feature that were already included a parent will have their Namespace updated as well
// (happens when Feature detectors relies on the detected layer Namespace). However, if the listed
// Feature has the same Name/Version as its parent, InsertLayer considers that the Feature hasn't
// been modified.
func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
tf := time.Now()
// Verify parameters
if layer.Name == "" {
log.Warning("could not insert a layer which has an empty Name")
return cerrors.NewBadRequestError("could not insert a layer which has an empty Name")
}
// Get a potentially existing layer.
existingLayer, err := pgSQL.FindLayer(layer.Name, true, false)
if err != nil && err != cerrors.ErrNotFound {
return err
} else if err == nil {
if existingLayer.EngineVersion >= layer.EngineVersion {
// The layer exists and has an equal or higher engine version, do nothing.
return nil
}
layer.ID = existingLayer.ID
}
// We do `defer observeQueryTime` here because we don't want to observe existing layers.
defer observeQueryTime("InsertLayer", "all", tf)
// Get parent ID.
var parentID zero.Int
if layer.Parent != nil {
if layer.Parent.ID == 0 {
log.Warning("Parent is expected to be retrieved from database when inserting a layer.")
return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.")
}
parentID = zero.IntFrom(int64(layer.Parent.ID))
}
// Find or insert namespace if provided.
var namespaceID zero.Int
if layer.Namespace != nil {
n, err := pgSQL.insertNamespace(*layer.Namespace)
if err != nil {
return err
}
namespaceID = zero.IntFrom(int64(n))
} else if layer.Namespace == nil && layer.Parent != nil {
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
if layer.Parent.Namespace != nil {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
}
}
// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return handleError("InsertLayer.Begin()", err)
}
if layer.ID == 0 {
// Insert a new layer.
err = tx.QueryRow(insertLayer, layer.Name, layer.EngineVersion, parentID, namespaceID).
Scan(&layer.ID)
if err != nil {
tx.Rollback()
if isErrUniqueViolation(err) {
// Ignore this error, another process collided.
log.Debug("Attempted to insert duplicate layer.")
return nil
}
return handleError("insertLayer", err)
}
} else {
// Update an existing layer.
_, err = tx.Exec(updateLayer, layer.ID, layer.EngineVersion, namespaceID)
if err != nil {
tx.Rollback()
return handleError("updateLayer", err)
}
// Remove all existing Layer_diff_FeatureVersion.
_, err = tx.Exec(removeLayerDiffFeatureVersion, layer.ID)
if err != nil {
tx.Rollback()
return handleError("removeLayerDiffFeatureVersion", err)
}
}
// Update Layer_diff_FeatureVersion now.
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
if err != nil {
tx.Rollback()
return err
}
// Commit transaction.
err = tx.Commit()
if err != nil {
tx.Rollback()
return handleError("InsertLayer.Commit()", err)
}
return nil
}
func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error {
// add and del are the FeatureVersion diff we should insert.
var add []database.FeatureVersion
var del []database.FeatureVersion
if layer.Parent == nil {
// There is no parent, every Features are added.
add = append(add, layer.Features...)
} else if layer.Parent != nil {
// There is a parent, we need to diff the Features with it.
// Build name:version structures.
layerFeaturesMapNV, layerFeaturesNV := createNV(layer.Features)
parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features)
// Calculate the added and deleted FeatureVersions name:version.
addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV)
delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV)
// Fill the structures containing the added and deleted FeatureVersions
for _, nv := range addNV {
add = append(add, *layerFeaturesMapNV[nv])
}
for _, nv := range delNV {
del = append(del, *parentLayerFeaturesMapNV[nv])
}
}
// Insert FeatureVersions in the database.
addIDs, err := pgSQL.insertFeatureVersions(add)
if err != nil {
return err
}
delIDs, err := pgSQL.insertFeatureVersions(del)
if err != nil {
return err
}
// Insert diff in the database.
if len(addIDs) > 0 {
_, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "add", buildInputArray(addIDs))
if err != nil {
return handleError("insertLayerDiffFeatureVersion.Add", err)
}
}
if len(delIDs) > 0 {
_, err = tx.Exec(insertLayerDiffFeatureVersion, layer.ID, "del", buildInputArray(delIDs))
if err != nil {
return handleError("insertLayerDiffFeatureVersion.Del", err)
}
}
return nil
}
func createNV(features []database.FeatureVersion) (map[string]*database.FeatureVersion, []string) {
mapNV := make(map[string]*database.FeatureVersion, 0)
sliceNV := make([]string, 0, len(features))
for i := 0; i < len(features); i++ {
featureVersion := &features[i]
nv := featureVersion.Feature.Name + ":" + featureVersion.Version.String()
mapNV[nv] = featureVersion
sliceNV = append(sliceNV, nv)
}
return mapNV, sliceNV
}
func (pgSQL *pgSQL) DeleteLayer(name string) error {
defer observeQueryTime("DeleteLayer", "all", time.Now())
result, err := pgSQL.Exec(removeLayer, name)
if err != nil {
return handleError("removeLayer", err)
}
affected, err := result.RowsAffected()
if err != nil {
return handleError("removeLayer.RowsAffected()", err)
}
if affected <= 0 {
return cerrors.ErrNotFound
}
return nil
}

@ -0,0 +1,105 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"time"
cerrors "github.com/coreos/clair/utils/errors"
)
// Lock tries to set a temporary lock in the database.
//
// Lock does not block, instead, it returns true and its expiration time
// is the lock has been successfully acquired or false otherwise
func (pgSQL *pgSQL) Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time) {
if name == "" || owner == "" || duration == 0 {
log.Warning("could not create an invalid lock")
return false, time.Time{}
}
defer observeQueryTime("Lock", "all", time.Now())
// Compute expiration.
until := time.Now().Add(duration)
if renew {
// Renew lock.
r, err := pgSQL.Exec(updateLock, name, owner, until)
if err != nil {
handleError("updateLock", err)
return false, until
}
if n, _ := r.RowsAffected(); n > 0 {
// Updated successfully.
return true, until
}
} else {
// Prune locks.
pgSQL.pruneLocks()
}
// Lock.
_, err := pgSQL.Exec(insertLock, name, owner, until)
if err != nil {
if !isErrUniqueViolation(err) {
handleError("insertLock", err)
}
return false, until
}
return true, until
}
// Unlock unlocks a lock specified by its name if I own it
func (pgSQL *pgSQL) Unlock(name, owner string) {
if name == "" || owner == "" {
log.Warning("could not delete an invalid lock")
return
}
defer observeQueryTime("Unlock", "all", time.Now())
pgSQL.Exec(removeLock, name, owner)
}
// FindLock returns the owner of a lock specified by its name and its
// expiration time.
func (pgSQL *pgSQL) FindLock(name string) (string, time.Time, error) {
if name == "" {
log.Warning("could not find an invalid lock")
return "", time.Time{}, cerrors.NewBadRequestError("could not find an invalid lock")
}
defer observeQueryTime("FindLock", "all", time.Now())
var owner string
var until time.Time
err := pgSQL.QueryRow(searchLock, name).Scan(&owner, &until)
if err != nil {
return owner, until, handleError("searchLock", err)
}
return owner, until, nil
}
// pruneLocks removes every expired locks from the database
func (pgSQL *pgSQL) pruneLocks() {
defer observeQueryTime("pruneLocks", "all", time.Now())
if _, err := pgSQL.Exec(removeLockExpired); err != nil {
handleError("removeLockExpired", err)
}
}

@ -0,0 +1,174 @@
-- Copyright 2015 clair authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-- +goose Up
-- -----------------------------------------------------
-- Table Namespace
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Namespace (
id SERIAL PRIMARY KEY,
name VARCHAR(128) NULL);
-- -----------------------------------------------------
-- Table Layer
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Layer (
id SERIAL PRIMARY KEY,
name VARCHAR(128) NOT NULL UNIQUE,
engineversion SMALLINT NOT NULL,
parent_id INT NULL REFERENCES Layer ON DELETE CASCADE,
namespace_id INT NULL REFERENCES Namespace,
created_at TIMESTAMP WITH TIME ZONE);
CREATE INDEX ON Layer (parent_id);
CREATE INDEX ON Layer (namespace_id);
-- -----------------------------------------------------
-- Table Feature
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Feature (
id SERIAL PRIMARY KEY,
namespace_id INT NOT NULL REFERENCES Namespace,
name VARCHAR(128) NOT NULL,
UNIQUE (namespace_id, name));
-- -----------------------------------------------------
-- Table FeatureVersion
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS FeatureVersion (
id SERIAL PRIMARY KEY,
feature_id INT NOT NULL REFERENCES Feature,
version VARCHAR(128) NOT NULL);
CREATE INDEX ON FeatureVersion (feature_id);
-- -----------------------------------------------------
-- Table Layer_diff_FeatureVersion
-- -----------------------------------------------------
CREATE TYPE modification AS ENUM ('add', 'del');
CREATE TABLE IF NOT EXISTS Layer_diff_FeatureVersion (
id SERIAL PRIMARY KEY,
layer_id INT NOT NULL REFERENCES Layer ON DELETE CASCADE,
featureversion_id INT NOT NULL REFERENCES FeatureVersion,
modification modification NOT NULL,
UNIQUE (layer_id, featureversion_id));
CREATE INDEX ON Layer_diff_FeatureVersion (layer_id);
CREATE INDEX ON Layer_diff_FeatureVersion (featureversion_id);
CREATE INDEX ON Layer_diff_FeatureVersion (featureversion_id, layer_id);
-- -----------------------------------------------------
-- Table Vulnerability
-- -----------------------------------------------------
CREATE TYPE severity AS ENUM ('Unknown', 'Negligible', 'Low', 'Medium', 'High', 'Critical', 'Defcon1');
CREATE TABLE IF NOT EXISTS Vulnerability (
id SERIAL PRIMARY KEY,
namespace_id INT NOT NULL REFERENCES Namespace,
name VARCHAR(128) NOT NULL,
description TEXT NULL,
link VARCHAR(128) NULL,
severity severity NOT NULL,
metadata TEXT NULL,
created_at TIMESTAMP WITH TIME ZONE,
deleted_at TIMESTAMP WITH TIME ZONE NULL);
-- -----------------------------------------------------
-- Table Vulnerability_FixedIn_Feature
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Vulnerability_FixedIn_Feature (
id SERIAL PRIMARY KEY,
vulnerability_id INT NOT NULL REFERENCES Vulnerability ON DELETE CASCADE,
feature_id INT NOT NULL REFERENCES Feature,
version VARCHAR(128) NOT NULL,
UNIQUE (vulnerability_id, feature_id));
CREATE INDEX ON Vulnerability_FixedIn_Feature (feature_id, vulnerability_id);
-- -----------------------------------------------------
-- Table Vulnerability_Affects_FeatureVersion
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Vulnerability_Affects_FeatureVersion (
id SERIAL PRIMARY KEY,
vulnerability_id INT NOT NULL REFERENCES Vulnerability ON DELETE CASCADE,
featureversion_id INT NOT NULL REFERENCES FeatureVersion,
fixedin_id INT NOT NULL REFERENCES Vulnerability_FixedIn_Feature ON DELETE CASCADE,
UNIQUE (vulnerability_id, featureversion_id));
CREATE INDEX ON Vulnerability_Affects_FeatureVersion (fixedin_id);
CREATE INDEX ON Vulnerability_Affects_FeatureVersion (featureversion_id, vulnerability_id);
-- -----------------------------------------------------
-- Table KeyValue
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS KeyValue (
id SERIAL PRIMARY KEY,
key VARCHAR(128) NOT NULL UNIQUE,
value TEXT);
-- -----------------------------------------------------
-- Table Lock
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Lock (
id SERIAL PRIMARY KEY,
name VARCHAR(64) NOT NULL UNIQUE,
owner VARCHAR(64) NOT NULL,
until TIMESTAMP WITH TIME ZONE);
CREATE INDEX ON Lock (owner);
-- -----------------------------------------------------
-- Table VulnerabilityNotification
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS Vulnerability_Notification (
id SERIAL PRIMARY KEY,
name VARCHAR(64) NOT NULL UNIQUE,
created_at TIMESTAMP WITH TIME ZONE,
notified_at TIMESTAMP WITH TIME ZONE NULL,
deleted_at TIMESTAMP WITH TIME ZONE NULL,
old_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE,
new_vulnerability_id INT NULL REFERENCES Vulnerability ON DELETE CASCADE);
CREATE INDEX ON Vulnerability_Notification (notified_at);
-- +goose Down
DROP TABLE IF EXISTS Namespace,
Layer,
Feature,
FeatureVersion,
Layer_diff_FeatureVersion,
Vulnerability,
Vulnerability_FixedIn_Feature,
Vulnerability_Affects_FeatureVersion,
Vulnerability_Notification,
KeyValue,
Lock
CASCADE;

@ -0,0 +1,75 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"time"
"github.com/coreos/clair/database"
cerrors "github.com/coreos/clair/utils/errors"
)
func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
if namespace.Name == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace")
}
if pgSQL.cache != nil {
promCacheQueriesTotal.WithLabelValues("namespace").Inc()
if id, found := pgSQL.cache.Get("namespace:" + namespace.Name); found {
promCacheHitsTotal.WithLabelValues("namespace").Inc()
return id.(int), nil
}
}
// We do `defer observeQueryTime` here because we don't want to observe cached namespaces.
defer observeQueryTime("insertNamespace", "all", time.Now())
var id int
err := pgSQL.QueryRow(soiNamespace, namespace.Name).Scan(&id)
if err != nil {
return 0, handleError("soiNamespace", err)
}
if pgSQL.cache != nil {
pgSQL.cache.Add("namespace:"+namespace.Name, id)
}
return id, nil
}
func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error) {
rows, err := pgSQL.Query(listNamespace)
if err != nil {
return namespaces, handleError("listNamespace", err)
}
defer rows.Close()
for rows.Next() {
var namespace database.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name)
if err != nil {
return namespaces, handleError("listNamespace.Scan()", err)
}
namespaces = append(namespaces, namespace)
}
if err = rows.Err(); err != nil {
return namespaces, handleError("listNamespace.Rows()", err)
}
return namespaces, err
}

@ -0,0 +1,214 @@
package pgsql
import (
"database/sql"
"time"
"github.com/coreos/clair/database"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/guregu/null/zero"
"github.com/pborman/uuid"
)
// do it in tx so we won't insert/update a vuln without notification and vice-versa.
// name and created doesn't matter.
func createNotification(tx *sql.Tx, oldVulnerabilityID, newVulnerabilityID int) error {
defer observeQueryTime("createNotification", "all", time.Now())
// Insert Notification.
oldVulnerabilityNullableID := sql.NullInt64{Int64: int64(oldVulnerabilityID), Valid: oldVulnerabilityID != 0}
newVulnerabilityNullableID := sql.NullInt64{Int64: int64(newVulnerabilityID), Valid: newVulnerabilityID != 0}
_, err := tx.Exec(insertNotification, uuid.New(), oldVulnerabilityNullableID, newVulnerabilityNullableID)
if err != nil {
tx.Rollback()
return handleError("insertNotification", err)
}
return nil
}
// Get one available notification name (!locked && !deleted && (!notified || notified_but_timed-out)).
// Does not fill new/old vuln.
func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (database.VulnerabilityNotification, error) {
defer observeQueryTime("GetAvailableNotification", "all", time.Now())
before := time.Now().Add(-renotifyInterval)
row := pgSQL.QueryRow(searchNotificationAvailable, before)
notification, err := pgSQL.scanNotification(row, false)
return notification, handleError("searchNotificationAvailable", err)
}
func (pgSQL *pgSQL) GetNotification(name string, limit int, page database.VulnerabilityNotificationPageNumber) (database.VulnerabilityNotification, database.VulnerabilityNotificationPageNumber, error) {
defer observeQueryTime("GetNotification", "all", time.Now())
// Get Notification.
notification, err := pgSQL.scanNotification(pgSQL.QueryRow(searchNotification, name), true)
if err != nil {
return notification, page, handleError("searchNotification", err)
}
// Load vulnerabilities' LayersIntroducingVulnerability.
page.OldVulnerability, err = pgSQL.loadLayerIntroducingVulnerability(
notification.OldVulnerability,
limit,
page.OldVulnerability,
)
if err != nil {
return notification, page, err
}
page.NewVulnerability, err = pgSQL.loadLayerIntroducingVulnerability(
notification.NewVulnerability,
limit,
page.NewVulnerability,
)
if err != nil {
return notification, page, err
}
return notification, page, nil
}
func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (database.VulnerabilityNotification, error) {
var notification database.VulnerabilityNotification
var created zero.Time
var notified zero.Time
var deleted zero.Time
var oldVulnerabilityNullableID sql.NullInt64
var newVulnerabilityNullableID sql.NullInt64
// Scan notification.
if hasVulns {
err := row.Scan(
&notification.ID,
&notification.Name,
&created,
&notified,
&deleted,
&oldVulnerabilityNullableID,
&newVulnerabilityNullableID,
)
if err != nil {
return notification, err
}
} else {
err := row.Scan(&notification.ID, &notification.Name, &created, &notified, &deleted)
if err != nil {
return notification, err
}
}
notification.Created = created.Time
notification.Notified = notified.Time
notification.Deleted = deleted.Time
if hasVulns {
if oldVulnerabilityNullableID.Valid {
vulnerability, err := pgSQL.findVulnerabilityByIDWithDeleted(int(oldVulnerabilityNullableID.Int64))
if err != nil {
return notification, err
}
notification.OldVulnerability = &vulnerability
}
if newVulnerabilityNullableID.Valid {
vulnerability, err := pgSQL.findVulnerabilityByIDWithDeleted(int(newVulnerabilityNullableID.Int64))
if err != nil {
return notification, err
}
notification.NewVulnerability = &vulnerability
}
}
return notification, nil
}
// Fills Vulnerability.LayersIntroducingVulnerability.
// limit -1: won't do anything
// limit 0: will just get the startID of the second page
func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *database.Vulnerability, limit, startID int) (int, error) {
tf := time.Now()
if vulnerability == nil {
return -1, nil
}
// A startID equals to -1 means that we reached the end already.
if startID == -1 || limit == -1 {
return -1, nil
}
// We do `defer observeQueryTime` here because we don't want to observe invalid calls.
defer observeQueryTime("loadLayerIntroducingVulnerability", "all", tf)
// Query with limit + 1, the last item will be used to know the next starting ID.
rows, err := pgSQL.Query(searchNotificationLayerIntroducingVulnerability,
vulnerability.ID, startID, limit+1)
if err != nil {
return 0, handleError("searchVulnerabilityFixedInFeature", err)
}
defer rows.Close()
var layers []database.Layer
for rows.Next() {
var layer database.Layer
if err := rows.Scan(&layer.ID, &layer.Name); err != nil {
return -1, handleError("searchNotificationLayerIntroducingVulnerability.Scan()", err)
}
layers = append(layers, layer)
}
if err = rows.Err(); err != nil {
return -1, handleError("searchNotificationLayerIntroducingVulnerability.Rows()", err)
}
size := limit
if len(layers) < limit {
size = len(layers)
}
vulnerability.LayersIntroducingVulnerability = layers[:size]
nextID := -1
if len(layers) > limit {
nextID = layers[limit].ID
}
return nextID, nil
}
func (pgSQL *pgSQL) SetNotificationNotified(name string) error {
defer observeQueryTime("SetNotificationNotified", "all", time.Now())
if _, err := pgSQL.Exec(updatedNotificationNotified, name); err != nil {
return handleError("updatedNotificationNotified", err)
}
return nil
}
func (pgSQL *pgSQL) DeleteNotification(name string) error {
defer observeQueryTime("DeleteNotification", "all", time.Now())
result, err := pgSQL.Exec(removeNotification, name)
if err != nil {
return handleError("removeNotification", err)
}
affected, err := result.RowsAffected()
if err != nil {
return handleError("removeNotification.RowsAffected()", err)
}
if affected <= 0 {
return cerrors.ErrNotFound
}
return nil
}

@ -0,0 +1,287 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pgsql implements database.Datastore with PostgreSQL.
package pgsql
import (
"database/sql"
"fmt"
"io/ioutil"
"os"
"path"
"runtime"
"strings"
"time"
"bitbucket.org/liamstask/goose/lib/goose"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/pkg/capnslog"
"github.com/hashicorp/golang-lru"
"github.com/lib/pq"
"github.com/pborman/uuid"
"github.com/prometheus/client_golang/prometheus"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "pgsql")
promErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "clair_pgsql_errors_total",
Help: "Number of errors that PostgreSQL requests generated.",
}, []string{"request"})
promCacheHitsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "clair_pgsql_cache_hits_total",
Help: "Number of cache hits that the PostgreSQL backend did.",
}, []string{"object"})
promCacheQueriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "clair_pgsql_cache_queries_total",
Help: "Number of cache queries that the PostgreSQL backend did.",
}, []string{"object"})
promQueryDurationMilliseconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "clair_pgsql_query_duration_milliseconds",
Help: "Time it takes to execute the database query.",
}, []string{"query", "subquery"})
promConcurrentLockVAFV = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "clair_pgsql_concurrent_lock_vafv_total",
Help: "Number of transactions trying to hold the exclusive Vulnerability_Affects_FeatureVersion lock.",
})
)
func init() {
prometheus.MustRegister(promErrorsTotal)
prometheus.MustRegister(promCacheHitsTotal)
prometheus.MustRegister(promCacheQueriesTotal)
prometheus.MustRegister(promQueryDurationMilliseconds)
prometheus.MustRegister(promConcurrentLockVAFV)
}
type Queryer interface {
Query(query string, args ...interface{}) (*sql.Rows, error)
QueryRow(query string, args ...interface{}) *sql.Row
}
type pgSQL struct {
*sql.DB
cache *lru.ARCCache
}
func (pgSQL *pgSQL) Close() {
pgSQL.DB.Close()
}
func (pgSQL *pgSQL) Ping() bool {
return pgSQL.DB.Ping() == nil
}
// Open creates a Datastore backed by a PostgreSQL database.
//
// It will run immediately every necessary migration on the database.
func Open(config *config.DatabaseConfig) (database.Datastore, error) {
// Run migrations.
if err := migrate(config.Source); err != nil {
log.Error(err)
return nil, database.ErrCantOpen
}
// Open database.
db, err := sql.Open("postgres", config.Source)
if err != nil {
log.Error(err)
return nil, database.ErrCantOpen
}
// Initialize cache.
// TODO(Quentin-M): Benchmark with a simple LRU Cache.
var cache *lru.ARCCache
if config.CacheSize > 0 {
cache, _ = lru.NewARC(config.CacheSize)
}
return &pgSQL{DB: db, cache: cache}, nil
}
// migrate runs all available migrations on a pgSQL database.
func migrate(dataSource string) error {
log.Info("running database migrations")
_, filename, _, _ := runtime.Caller(1)
migrationDir := path.Join(path.Dir(filename), "/migrations/")
conf := &goose.DBConf{
MigrationsDir: migrationDir,
Driver: goose.DBDriver{
Name: "postgres",
OpenStr: dataSource,
Import: "github.com/lib/pq",
Dialect: &goose.PostgresDialect{},
},
}
// Determine the most recent revision available from the migrations folder.
target, err := goose.GetMostRecentDBVersion(conf.MigrationsDir)
if err != nil {
return err
}
// Run migrations
err = goose.RunMigrations(conf, conf.MigrationsDir, target)
if err != nil {
return err
}
log.Info("database migration ran successfully")
return nil
}
// createDatabase creates a new database.
// The dataSource parameter should not contain a dbname.
func createDatabase(dataSource, databaseName string) error {
// Open database.
db, err := sql.Open("postgres", dataSource)
if err != nil {
return fmt.Errorf("could not open database (CreateDatabase): %v", err)
}
defer db.Close()
// Create database.
_, err = db.Exec("CREATE DATABASE " + databaseName)
if err != nil {
return fmt.Errorf("could not create database: %v", err)
}
return nil
}
// dropDatabase drops an existing database.
// The dataSource parameter should not contain a dbname.
func dropDatabase(dataSource, databaseName string) error {
// Open database.
db, err := sql.Open("postgres", dataSource)
if err != nil {
return fmt.Errorf("could not open database (DropDatabase): %v", err)
}
defer db.Close()
// Kill any opened connection.
if _, err := db.Exec(`
SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
WHERE pg_stat_activity.datname = $1
AND pid <> pg_backend_pid()`, databaseName); err != nil {
return fmt.Errorf("could not drop database: %v", err)
}
// Drop database.
if _, err = db.Exec("DROP DATABASE " + databaseName); err != nil {
return fmt.Errorf("could not drop database: %v", err)
}
return nil
}
// pgSQLTest wraps pgSQL for testing purposes.
// Its Close() method drops the database.
type pgSQLTest struct {
*pgSQL
dataSourceDefaultDatabase string
dbName string
}
// OpenForTest creates a test Datastore backed by a new PostgreSQL database.
// It creates a new unique and prefixed ("test_") database.
// Using Close() will drop the database.
func OpenForTest(name string, withTestData bool) (*pgSQLTest, error) {
// Define the PostgreSQL connection strings.
dataSource := "host=127.0.0.1 sslmode=disable user=postgres dbname="
if dataSourceEnv := os.Getenv("CLAIR_TEST_PGSQL"); dataSourceEnv != "" {
dataSource = dataSourceEnv + " dbname="
}
dbName := "test_" + strings.ToLower(name) + "_" + strings.Replace(uuid.New(), "-", "_", -1)
dataSourceDefaultDatabase := dataSource + "postgres"
dataSourceTestDatabase := dataSource + dbName
// Create database.
if err := createDatabase(dataSourceDefaultDatabase, dbName); err != nil {
log.Error(err)
return nil, database.ErrCantOpen
}
// Open database.
db, err := Open(&config.DatabaseConfig{Source: dataSourceTestDatabase, CacheSize: 0})
if err != nil {
dropDatabase(dataSourceDefaultDatabase, dbName)
log.Error(err)
return nil, database.ErrCantOpen
}
// Load test data if specified.
if withTestData {
_, filename, _, _ := runtime.Caller(0)
d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/testdata/data.sql")
_, err = db.(*pgSQL).Exec(string(d))
if err != nil {
dropDatabase(dataSourceDefaultDatabase, dbName)
log.Error(err)
return nil, database.ErrCantOpen
}
}
return &pgSQLTest{
pgSQL: db.(*pgSQL),
dataSourceDefaultDatabase: dataSourceDefaultDatabase,
dbName: dbName}, nil
}
func (pgSQL *pgSQLTest) Close() {
pgSQL.DB.Close()
dropDatabase(pgSQL.dataSourceDefaultDatabase, pgSQL.dbName)
}
// handleError logs an error with an extra description and masks the error if it's an SQL one.
// This ensures we never return plain SQL errors and leak anything.
func handleError(desc string, err error) error {
if err == nil {
return nil
}
if err == sql.ErrNoRows {
return cerrors.ErrNotFound
}
log.Errorf("%s: %v", desc, err)
promErrorsTotal.WithLabelValues(desc).Inc()
if _, o := err.(*pq.Error); o || err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") {
return database.ErrBackendException
}
return err
}
// isErrUniqueViolation determines is the given error is a unique contraint violation.
func isErrUniqueViolation(err error) bool {
pqErr, ok := err.(*pq.Error)
return ok && pqErr.Code == "23505"
}
func observeQueryTime(query, subquery string, start time.Time) {
utils.PrometheusObserveTimeMilliseconds(promQueryDurationMilliseconds.WithLabelValues(query, subquery), start)
}

@ -0,0 +1,239 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import "strconv"
const (
lockVulnerabilityAffects = `LOCK Vulnerability_Affects_FeatureVersion IN SHARE ROW EXCLUSIVE MODE`
disableHashJoin = `SET LOCAL enable_hashjoin = off`
disableMergeJoin = `SET LOCAL enable_mergejoin = off`
// keyvalue.go
updateKeyValue = `UPDATE KeyValue SET value = $1 WHERE key = $2`
insertKeyValue = `INSERT INTO KeyValue(key, value) VALUES($1, $2)`
searchKeyValue = `SELECT value FROM KeyValue WHERE key = $1`
// namespace.go
soiNamespace = `
WITH new_namespace AS (
INSERT INTO Namespace(name)
SELECT CAST($1 AS VARCHAR)
WHERE NOT EXISTS (SELECT name FROM Namespace WHERE name = $1)
RETURNING id
)
SELECT id FROM Namespace WHERE name = $1
UNION
SELECT id FROM new_namespace`
searchNamespace = `SELECT id FROM Namespace WHERE name = $1`
listNamespace = `SELECT id, name FROM Namespace`
// feature.go
soiFeature = `
WITH new_feature AS (
INSERT INTO Feature(name, namespace_id)
SELECT CAST($1 AS VARCHAR), CAST($2 AS INTEGER)
WHERE NOT EXISTS (SELECT id FROM Feature WHERE name = $1 AND namespace_id = $2)
RETURNING id
)
SELECT id FROM Feature WHERE name = $1 AND namespace_id = $2
UNION
SELECT id FROM new_feature`
searchFeatureVersion = `
SELECT id FROM FeatureVersion WHERE feature_id = $1 AND version = $2`
soiFeatureVersion = `
WITH new_featureversion AS (
INSERT INTO FeatureVersion(feature_id, version)
SELECT CAST($1 AS INTEGER), CAST($2 AS VARCHAR)
WHERE NOT EXISTS (SELECT id FROM FeatureVersion WHERE feature_id = $1 AND version = $2)
RETURNING id
)
SELECT 'exi', id FROM FeatureVersion WHERE feature_id = $1 AND version = $2
UNION
SELECT 'new', id FROM new_featureversion`
searchVulnerabilityFixedInFeature = `
SELECT id, vulnerability_id, version FROM Vulnerability_FixedIn_Feature
WHERE feature_id = $1`
insertVulnerabilityAffectsFeatureVersion = `
INSERT INTO Vulnerability_Affects_FeatureVersion(vulnerability_id,
featureversion_id, fixedin_id) VALUES($1, $2, $3)`
// layer.go
searchLayer = `
SELECT l.id, l.name, l.engineversion, p.id, p.name, n.id, n.name
FROM Layer l
LEFT JOIN Layer p ON l.parent_id = p.id
LEFT JOIN Namespace n ON l.namespace_id = n.id
WHERE l.name = $1;`
searchLayerFeatureVersion = `
WITH RECURSIVE layer_tree(id, name, parent_id, depth, path, cycle) AS(
SELECT l.id, l.name, l.parent_id, 1, ARRAY[l.id], false
FROM Layer l
WHERE l.id = $1
UNION ALL
SELECT l.id, l.name, l.parent_id, lt.depth + 1, path || l.id, l.id = ANY(path)
FROM Layer l, layer_tree lt
WHERE l.id = lt.parent_id
)
SELECT ldf.featureversion_id, ldf.modification, fn.id, fn.name, f.id, f.name, fv.id, fv.version, ltree.id, ltree.name
FROM Layer_diff_FeatureVersion ldf
JOIN (
SELECT row_number() over (ORDER BY depth DESC), id, name FROM layer_tree
) AS ltree (ordering, id, name) ON ldf.layer_id = ltree.id, FeatureVersion fv, Feature f, Namespace fn
WHERE ldf.featureversion_id = fv.id AND fv.feature_id = f.id AND f.namespace_id = fn.id
ORDER BY ltree.ordering`
searchFeatureVersionVulnerability = `
SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata,
vn.name, vfif.version
FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v,
Namespace vn, Vulnerability_FixedIn_Feature vfif
WHERE vafv.featureversion_id = ANY($1::integer[])
AND vfif.vulnerability_id = v.id
AND vafv.fixedin_id = vfif.id
AND v.namespace_id = vn.id
AND v.deleted_at IS NULL`
insertLayer = `
INSERT INTO Layer(name, engineversion, parent_id, namespace_id, created_at)
VALUES($1, $2, $3, $4, CURRENT_TIMESTAMP)
RETURNING id`
updateLayer = `UPDATE LAYER SET engineversion = $2, namespace_id = $3 WHERE id = $1`
removeLayerDiffFeatureVersion = `
DELETE FROM Layer_diff_FeatureVersion
WHERE layer_id = $1`
insertLayerDiffFeatureVersion = `
INSERT INTO Layer_diff_FeatureVersion(layer_id, featureversion_id, modification)
SELECT $1, fv.id, $2
FROM FeatureVersion fv
WHERE fv.id = ANY($3::integer[])`
removeLayer = `DELETE FROM Layer WHERE name = $1`
// lock.go
insertLock = `INSERT INTO Lock(name, owner, until) VALUES($1, $2, $3)`
searchLock = `SELECT owner, until FROM Lock WHERE name = $1`
updateLock = `UPDATE Lock SET until = $3 WHERE name = $1 AND owner = $2`
removeLock = `DELETE FROM Lock WHERE name = $1 AND owner = $2`
removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP`
// vulnerability.go
searchVulnerabilityBase = `
SELECT v.id, v.name, n.id, n.name, v.description, v.link, v.severity, v.metadata
FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id`
searchVulnerabilityForUpdate = ` FOR UPDATE OF v`
searchVulnerabilityByNamespaceAndName = ` WHERE n.name = $1 AND v.name = $2 AND v.deleted_at IS NULL`
searchVulnerabilityByID = ` WHERE v.id = $1`
searchVulnerabilityByNamespace = ` WHERE n.name = $1 AND v.deleted_at IS NULL
AND v.id >= $2
ORDER BY v.id
LIMIT $3`
searchVulnerabilityFixedIn = `
SELECT vfif.version, f.id, f.Name
FROM Vulnerability_FixedIn_Feature vfif JOIN Feature f ON vfif.feature_id = f.id
WHERE vfif.vulnerability_id = $1`
insertVulnerability = `
INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata, created_at)
VALUES($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP)
RETURNING id`
insertVulnerabilityFixedInFeature = `
INSERT INTO Vulnerability_FixedIn_Feature(vulnerability_id, feature_id, version)
VALUES($1, $2, $3)
RETURNING id`
searchFeatureVersionByFeature = `SELECT id, version FROM FeatureVersion WHERE feature_id = $1`
removeVulnerability = `
UPDATE Vulnerability
SET deleted_at = CURRENT_TIMESTAMP
WHERE namespace_id = (SELECT id FROM Namespace WHERE name = $1)
AND name = $2
AND deleted_at IS NULL
RETURNING id`
// notification.go
insertNotification = `
INSERT INTO Vulnerability_Notification(name, created_at, old_vulnerability_id, new_vulnerability_id)
VALUES($1, CURRENT_TIMESTAMP, $2, $3)`
updatedNotificationNotified = `
UPDATE Vulnerability_Notification
SET notified_at = CURRENT_TIMESTAMP
WHERE name = $1`
removeNotification = `
UPDATE Vulnerability_Notification
SET deleted_at = CURRENT_TIMESTAMP
WHERE name = $1`
searchNotificationAvailable = `
SELECT id, name, created_at, notified_at, deleted_at
FROM Vulnerability_Notification
WHERE (notified_at IS NULL OR notified_at < $1)
AND deleted_at IS NULL
AND name NOT IN (SELECT name FROM Lock)
ORDER BY Random()
LIMIT 1`
searchNotification = `
SELECT id, name, created_at, notified_at, deleted_at, old_vulnerability_id, new_vulnerability_id
FROM Vulnerability_Notification
WHERE name = $1`
searchNotificationLayerIntroducingVulnerability = `
SELECT l.ID, l.name
FROM Vulnerability v, Vulnerability_Affects_FeatureVersion vafv, FeatureVersion fv, Layer_diff_FeatureVersion ldfv, Layer l
WHERE v.id = $1
AND v.id = vafv.vulnerability_id
AND vafv.featureversion_id = fv.id
AND fv.id = ldfv.featureversion_id
AND ldfv.modification = 'add'
AND ldfv.layer_id = l.id
AND l.id >= $2
ORDER BY l.ID
LIMIT $3`
// complex_test.go
searchComplexTestFeatureVersionAffects = `
SELECT v.name
FROM FeatureVersion fv
LEFT JOIN Vulnerability_Affects_FeatureVersion vaf ON fv.id = vaf.featureversion_id
JOIN Vulnerability v ON vaf.vulnerability_id = v.id
WHERE featureversion_id = $1`
)
// buildInputArray constructs a PostgreSQL input array from the specified integers.
// Useful to use the `= ANY($1::integer[])` syntax that let us use a IN clause while using
// a single placeholder.
func buildInputArray(ints []int) string {
str := "{"
for i := 0; i < len(ints)-1; i++ {
str = str + strconv.Itoa(ints[i]) + ","
}
str = str + strconv.Itoa(ints[len(ints)-1]) + "}"
return str
}

@ -0,0 +1,569 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pgsql
import (
"database/sql"
"encoding/json"
"fmt"
"reflect"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/guregu/null/zero"
)
func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) {
defer observeQueryTime("listVulnerabilities", "all", time.Now())
// Query Namespace.
var id int
err := pgSQL.QueryRow(searchNamespace, namespaceName).Scan(&id)
if err != nil {
return nil, -1, handleError("searchNamespace", err)
} else if id == 0 {
return nil, -1, cerrors.ErrNotFound
}
// Query.
query := searchVulnerabilityBase + searchVulnerabilityByNamespace
rows, err := pgSQL.Query(query, namespaceName, startID, limit+1)
if err != nil {
return nil, -1, handleError("searchVulnerabilityByNamespace", err)
}
defer rows.Close()
var vulns []database.Vulnerability
nextID := -1
size := 0
// Scan query.
for rows.Next() {
var vulnerability database.Vulnerability
err := rows.Scan(
&vulnerability.ID,
&vulnerability.Name,
&vulnerability.Namespace.ID,
&vulnerability.Namespace.Name,
&vulnerability.Description,
&vulnerability.Link,
&vulnerability.Severity,
&vulnerability.Metadata,
)
if err != nil {
return nil, -1, handleError("searchVulnerabilityByNamespace.Scan()", err)
}
size++
if size > limit {
nextID = vulnerability.ID
} else {
vulns = append(vulns, vulnerability)
}
}
if err := rows.Err(); err != nil {
return nil, -1, handleError("searchVulnerabilityByNamespace.Rows()", err)
}
return vulns, nextID, nil
}
func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) {
return findVulnerability(pgSQL, namespaceName, name, false)
}
func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) {
defer observeQueryTime("findVulnerability", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName"
query := searchVulnerabilityBase + searchVulnerabilityByNamespaceAndName
if forUpdate {
queryName = queryName + "+searchVulnerabilityForUpdate"
query = query + searchVulnerabilityForUpdate
}
return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name))
}
func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) {
defer observeQueryTime("findVulnerabilityByIDWithDeleted", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByID"
query := searchVulnerabilityBase + searchVulnerabilityByID
return scanVulnerability(pgSQL, queryName, pgSQL.QueryRow(query, id))
}
func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.Row) (database.Vulnerability, error) {
var vulnerability database.Vulnerability
err := vulnerabilityRow.Scan(
&vulnerability.ID,
&vulnerability.Name,
&vulnerability.Namespace.ID,
&vulnerability.Namespace.Name,
&vulnerability.Description,
&vulnerability.Link,
&vulnerability.Severity,
&vulnerability.Metadata,
)
if err != nil {
return vulnerability, handleError(queryName+".Scan()", err)
}
if vulnerability.ID == 0 {
return vulnerability, cerrors.ErrNotFound
}
// Query the FixedIn FeatureVersion now.
rows, err := queryer.Query(searchVulnerabilityFixedIn, vulnerability.ID)
if err != nil {
return vulnerability, handleError("searchVulnerabilityFixedIn.Scan()", err)
}
defer rows.Close()
for rows.Next() {
var featureVersionID zero.Int
var featureVersionVersion zero.String
var featureVersionFeatureName zero.String
err := rows.Scan(
&featureVersionVersion,
&featureVersionID,
&featureVersionFeatureName,
)
if err != nil {
return vulnerability, handleError("searchVulnerabilityFixedIn.Scan()", err)
}
if !featureVersionID.IsZero() {
// Note that the ID we fill in featureVersion is actually a Feature ID, and not
// a FeatureVersion ID.
featureVersion := database.FeatureVersion{
Model: database.Model{ID: int(featureVersionID.Int64)},
Feature: database.Feature{
Model: database.Model{ID: int(featureVersionID.Int64)},
Namespace: vulnerability.Namespace,
Name: featureVersionFeatureName.String,
},
Version: types.NewVersionUnsafe(featureVersionVersion.String),
}
vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
}
}
if err := rows.Err(); err != nil {
return vulnerability, handleError("searchVulnerabilityFixedIn.Rows()", err)
}
return vulnerability, nil
}
// FixedIn.Namespace are not necessary, they are overwritten by the vuln.
// By setting the fixed version to minVersion, we can say that the vuln does'nt affect anymore.
func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerability, generateNotifications bool) error {
for _, vulnerability := range vulnerabilities {
err := pgSQL.insertVulnerability(vulnerability, false, generateNotifications)
if err != nil {
fmt.Printf("%#v\n", vulnerability)
return err
}
}
return nil
}
func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, onlyFixedIn, generateNotification bool) error {
tf := time.Now()
// Verify parameters
if vulnerability.Name == "" || vulnerability.Namespace.Name == "" {
return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace")
}
if !onlyFixedIn && !vulnerability.Severity.IsValid() {
msg := fmt.Sprintf("could not insert a vulnerability that has an invalid Severity: %s", vulnerability.Severity)
log.Warning(msg)
return cerrors.NewBadRequestError(msg)
}
for i := 0; i < len(vulnerability.FixedIn); i++ {
fifv := &vulnerability.FixedIn[i]
if fifv.Feature.Namespace.Name == "" {
// As there is no Namespace on that FixedIn FeatureVersion, set it to the Vulnerability's
// Namespace.
fifv.Feature.Namespace.Name = vulnerability.Namespace.Name
} else if fifv.Feature.Namespace.Name != vulnerability.Namespace.Name {
msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability"
log.Warning(msg)
return cerrors.NewBadRequestError(msg)
}
}
// We do `defer observeQueryTime` here because we don't want to observe invalid vulnerabilities.
defer observeQueryTime("insertVulnerability", "all", tf)
// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return handleError("insertVulnerability.Begin()", err)
}
// Find existing vulnerability and its Vulnerability_FixedIn_Features (for update).
existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace.Name, vulnerability.Name, true)
if err != nil && err != cerrors.ErrNotFound {
tx.Rollback()
return err
}
if onlyFixedIn {
// Because this call tries to update FixedIn FeatureVersion, import all other data from the
// existing one.
if existingVulnerability.ID == 0 {
return cerrors.ErrNotFound
}
fixedIn := vulnerability.FixedIn
vulnerability = existingVulnerability
vulnerability.FixedIn = fixedIn
}
if existingVulnerability.ID != 0 {
updateMetadata := vulnerability.Description != existingVulnerability.Description ||
vulnerability.Link != existingVulnerability.Link ||
vulnerability.Severity != existingVulnerability.Severity ||
!reflect.DeepEqual(castMetadata(vulnerability.Metadata), existingVulnerability.Metadata)
// Construct the entire list of FixedIn FeatureVersion, by using the
// the FixedIn list of the old vulnerability.
//
// TODO(Quentin-M): We could use !updateFixedIn to just copy FixedIn/Affects rows from the
// existing vulnerability in order to make metadata updates much faster.
var updateFixedIn bool
vulnerability.FixedIn, updateFixedIn = applyFixedInDiff(existingVulnerability.FixedIn, vulnerability.FixedIn)
if !updateMetadata && !updateFixedIn {
tx.Commit()
return nil
}
// Mark the old vulnerability as non latest.
_, err = tx.Exec(removeVulnerability, vulnerability.Namespace.Name, vulnerability.Name)
if err != nil {
tx.Rollback()
return handleError("removeVulnerability", err)
}
} else {
// The vulnerability is new, we don't want to have any types.MinVersion as they are only used
// for diffing existing vulnerabilities.
var fixedIn []database.FeatureVersion
for _, fv := range vulnerability.FixedIn {
if fv.Version != types.MinVersion {
fixedIn = append(fixedIn, fv)
}
}
vulnerability.FixedIn = fixedIn
}
// Find or insert Vulnerability's Namespace.
namespaceID, err := pgSQL.insertNamespace(vulnerability.Namespace)
if err != nil {
return err
}
// Insert vulnerability.
err = tx.QueryRow(
insertVulnerability,
namespaceID,
vulnerability.Name,
vulnerability.Description,
vulnerability.Link,
&vulnerability.Severity,
&vulnerability.Metadata,
).Scan(&vulnerability.ID)
if err != nil {
tx.Rollback()
return handleError("insertVulnerability", err)
}
// Update Vulnerability_FixedIn_Feature and Vulnerability_Affects_FeatureVersion now.
err = pgSQL.insertVulnerabilityFixedInFeatureVersions(tx, vulnerability.ID, vulnerability.FixedIn)
if err != nil {
tx.Rollback()
return err
}
// Create a notification.
if generateNotification {
err = createNotification(tx, existingVulnerability.ID, vulnerability.ID)
if err != nil {
return err
}
}
// Commit transaction.
err = tx.Commit()
if err != nil {
tx.Rollback()
return handleError("insertVulnerability.Commit()", err)
}
return nil
}
// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that
// everything has the interface{} type.
// It is required when comparing crafted MetadataMap against MetadataMap that we get from the
// database.
func castMetadata(m database.MetadataMap) database.MetadataMap {
c := make(database.MetadataMap)
j, _ := json.Marshal(m)
json.Unmarshal(j, &c)
return c
}
// applyFixedInDiff applies a FeatureVersion diff on a FeatureVersion list and returns the result.
func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.FeatureVersion, bool) {
currentMap, currentNames := createFeatureVersionNameMap(currentList)
diffMap, diffNames := createFeatureVersionNameMap(diff)
addedNames := utils.CompareStringLists(diffNames, currentNames)
inBothNames := utils.CompareStringListsInBoth(diffNames, currentNames)
different := false
for _, name := range addedNames {
if diffMap[name].Version == types.MinVersion {
// MinVersion only makes sense when a Feature is already fixed in some version,
// in which case we would be in the "inBothNames".
continue
}
currentMap[name] = diffMap[name]
different = true
}
for _, name := range inBothNames {
fv := diffMap[name]
if fv.Version == types.MinVersion {
// MinVersion means that the Feature doesn't affect the Vulnerability anymore.
delete(currentMap, name)
different = true
} else if fv.Version != currentMap[name].Version {
// The version got updated.
currentMap[name] = diffMap[name]
different = true
}
}
// Convert currentMap to a slice and return it.
var newList []database.FeatureVersion
for _, fv := range currentMap {
newList = append(newList, fv)
}
return newList, different
}
func createFeatureVersionNameMap(features []database.FeatureVersion) (map[string]database.FeatureVersion, []string) {
m := make(map[string]database.FeatureVersion, 0)
s := make([]string, 0, len(features))
for i := 0; i < len(features); i++ {
featureVersion := features[i]
m[featureVersion.Feature.Name] = featureVersion
s = append(s, featureVersion.Feature.Name)
}
return m, s
}
// insertVulnerabilityFixedInFeatureVersions populates Vulnerability_FixedIn_Feature for the given
// vulnerability with the specified database.FeatureVersion list and uses
// linkVulnerabilityToFeatureVersions to propagate the changes on Vulnerability_FixedIn_Feature to
// Vulnerability_Affects_FeatureVersion.
func (pgSQL *pgSQL) insertVulnerabilityFixedInFeatureVersions(tx *sql.Tx, vulnerabilityID int, fixedIn []database.FeatureVersion) error {
defer observeQueryTime("insertVulnerabilityFixedInFeatureVersions", "all", time.Now())
// Insert or find the Features.
// TODO(Quentin-M): Batch me.
var err error
var features []*database.Feature
for i := 0; i < len(fixedIn); i++ {
features = append(features, &fixedIn[i].Feature)
}
for _, feature := range features {
if feature.ID == 0 {
if feature.ID, err = pgSQL.insertFeature(*feature); err != nil {
return err
}
}
}
// Lock Vulnerability_Affects_FeatureVersion exclusively.
// We want to prevent InsertFeatureVersion to modify it.
promConcurrentLockVAFV.Inc()
defer promConcurrentLockVAFV.Dec()
t := time.Now()
_, err = tx.Exec(lockVulnerabilityAffects)
observeQueryTime("insertVulnerability", "lock", t)
if err != nil {
tx.Rollback()
return handleError("insertVulnerability.lockVulnerabilityAffects", err)
}
for _, fv := range fixedIn {
var fixedInID int
// Insert Vulnerability_FixedIn_Feature.
err = tx.QueryRow(
insertVulnerabilityFixedInFeature,
vulnerabilityID, fv.Feature.ID,
&fv.Version,
).Scan(&fixedInID)
if err != nil {
return handleError("insertVulnerabilityFixedInFeature", err)
}
// Insert Vulnerability_Affects_FeatureVersion.
err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerabilityID, fv.Feature.ID, fv.Version)
if err != nil {
return err
}
}
return nil
}
func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID, featureID int, fixedInVersion types.Version) error {
// Find every FeatureVersions of the Feature that the vulnerability affects.
// TODO(Quentin-M): LIMIT
rows, err := tx.Query(searchFeatureVersionByFeature, featureID)
if err != nil {
return handleError("searchFeatureVersionByFeature", err)
}
defer rows.Close()
var affecteds []database.FeatureVersion
for rows.Next() {
var affected database.FeatureVersion
err := rows.Scan(&affected.ID, &affected.Version)
if err != nil {
return handleError("searchFeatureVersionByFeature.Scan()", err)
}
if affected.Version.Compare(fixedInVersion) < 0 {
// The version of the FeatureVersion is lower than the fixed version of this vulnerability,
// thus, this FeatureVersion is affected by it.
affecteds = append(affecteds, affected)
}
}
if err = rows.Err(); err != nil {
return handleError("searchFeatureVersionByFeature.Rows()", err)
}
rows.Close()
// Insert into Vulnerability_Affects_FeatureVersion.
for _, affected := range affecteds {
// TODO(Quentin-M): Batch me.
_, err := tx.Exec(insertVulnerabilityAffectsFeatureVersion, vulnerabilityID,
affected.ID, fixedInID)
if err != nil {
return handleError("insertVulnerabilityAffectsFeatureVersion", err)
}
}
return nil
}
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error {
defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now())
v := database.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
},
FixedIn: fixes,
}
return pgSQL.insertVulnerability(v, true, true)
}
func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error {
defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now())
v := database.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
},
FixedIn: []database.FeatureVersion{
{
Feature: database.Feature{
Name: featureName,
Namespace: database.Namespace{
Name: vulnerabilityNamespace,
},
},
Version: types.MinVersion,
},
},
}
return pgSQL.insertVulnerability(v, true, true)
}
func (pgSQL *pgSQL) DeleteVulnerability(namespaceName, name string) error {
defer observeQueryTime("DeleteVulnerability", "all", time.Now())
// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return handleError("DeleteVulnerability.Begin()", err)
}
var vulnerabilityID int
err = tx.QueryRow(removeVulnerability, namespaceName, name).Scan(&vulnerabilityID)
if err != nil {
tx.Rollback()
return handleError("removeVulnerability", err)
}
// Create a notification.
err = createNotification(tx, vulnerabilityID, 0)
if err != nil {
return err
}
// Commit transaction.
err = tx.Commit()
if err != nil {
tx.Rollback()
return handleError("DeleteVulnerability.Commit()", err)
}
return nil
}

@ -0,0 +1,46 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package errors defines error types that are used in several modules
package errors
import "errors"
var (
// ErrFilesystem occurs when a filesystem interaction fails.
ErrFilesystem = errors.New("something went wrong when interacting with the fs")
// ErrCouldNotDownload occurs when a download fails.
ErrCouldNotDownload = errors.New("could not download requested ressource")
// ErrNotFound occurs when a resource could not be found.
ErrNotFound = errors.New("the resource cannot be found")
// ErrCouldNotParse is returned when a fetcher fails to parse the update data.
ErrCouldNotParse = errors.New("updater/fetchers: could not parse")
)
// ErrBadRequest occurs when a method has been passed an inappropriate argument.
type ErrBadRequest struct {
s string
}
// NewBadRequestError instantiates a ErrBadRequest with the specified message.
func NewBadRequestError(message string) error {
return &ErrBadRequest{s: message}
}
func (e *ErrBadRequest) Error() string {
return e.s
}

@ -0,0 +1,39 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package utils simply defines utility functions and types.
package utils
import (
"bytes"
"os/exec"
)
// Exec runs the given binary with arguments
func Exec(dir string, bin string, args ...string) ([]byte, error) {
_, err := exec.LookPath(bin)
if err != nil {
return nil, err
}
cmd := exec.Command(bin, args...)
cmd.Dir = dir
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Stderr = &buf
err = cmd.Run()
return buf.Bytes(), err
}

@ -0,0 +1,77 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package http provides utility functions for HTTP servers and clients.
package http
import (
"encoding/json"
"io"
"net/http"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker"
)
// MaxBodySize is the maximum number of bytes that ParseHTTPBody reads from an http.Request.Body.
const MaxBodySize int64 = 1048576
// WriteHTTP writes a JSON-encoded object to a http.ResponseWriter, as well as
// a HTTP status code.
func WriteHTTP(w http.ResponseWriter, httpStatus int, v interface{}) {
w.WriteHeader(httpStatus)
if v != nil {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
result, _ := json.Marshal(v)
w.Write(result)
}
}
// WriteHTTPError writes an error, wrapped in the Message field of a JSON-encoded
// object to a http.ResponseWriter, as well as a HTTP status code.
// If the status code is 0, handleError tries to guess the proper HTTP status
// code from the error type.
func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) {
if httpStatus == 0 {
httpStatus = http.StatusInternalServerError
// Try to guess the http status code from the error type
if _, isBadRequestError := err.(*cerrors.ErrBadRequest); isBadRequestError {
httpStatus = http.StatusBadRequest
} else {
switch err {
case cerrors.ErrNotFound:
httpStatus = http.StatusNotFound
case database.ErrBackendException:
httpStatus = http.StatusServiceUnavailable
case worker.ErrParentUnknown, worker.ErrUnsupported, utils.ErrCouldNotExtract, utils.ErrExtractedFileTooBig:
httpStatus = http.StatusBadRequest
}
}
}
WriteHTTP(w, httpStatus, struct{ Message string }{Message: err.Error()})
}
// ParseHTTPBody reads a JSON-encoded body from a http.Request and unmarshals it
// into the provided object.
func ParseHTTPBody(r *http.Request, v interface{}) (int, error) {
defer r.Body.Close()
err := json.NewDecoder(io.LimitReader(r.Body, MaxBodySize)).Decode(v)
if err != nil {
return http.StatusUnsupportedMediaType, err
}
return 0, nil
}

@ -0,0 +1,13 @@
package utils
import (
"time"
"github.com/prometheus/client_golang/prometheus"
)
// PrometheusObserveTimeMilliseconds observes the elapsed time since start, in milliseconds,
// on the specified Prometheus Histogram.
func PrometheusObserveTimeMilliseconds(h prometheus.Histogram, start time.Time) {
h.Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond))
}

@ -0,0 +1,65 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"sync"
"time"
)
// Stopper eases the graceful termination of a group of goroutines
type Stopper struct {
wg sync.WaitGroup
stop chan struct{}
}
// NewStopper initializes a new Stopper instance
func NewStopper() *Stopper {
return &Stopper{stop: make(chan struct{}, 0)}
}
// Begin indicates that a new goroutine has started.
func (s *Stopper) Begin() {
s.wg.Add(1)
}
// End indicates that a goroutine has stopped.
func (s *Stopper) End() {
s.wg.Done()
}
// Chan returns the channel on which goroutines could listen to determine if
// they should stop. The channel is closed when Stop() is called.
func (s *Stopper) Chan() chan struct{} {
return s.stop
}
// Sleep puts the current goroutine on sleep during a duration d
// Sleep could be interrupted in the case the goroutine should stop itself,
// in which case Sleep returns false.
func (s *Stopper) Sleep(d time.Duration) bool {
select {
case <-time.After(d):
return true
case <-s.stop:
return false
}
}
// Stop asks every goroutine to end.
func (s *Stopper) Stop() {
close(s.stop)
s.wg.Wait()
}

@ -0,0 +1,75 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import "regexp"
var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`)
// CleanURL removes all parameters from an URL.
func CleanURL(str string) string {
return urlParametersRegexp.ReplaceAllString(str, "")
}
// Contains looks for a string into an array of strings and returns whether
// the string exists.
func Contains(needle string, haystack []string) bool {
for _, h := range haystack {
if h == needle {
return true
}
}
return false
}
// CompareStringLists returns the strings that are present in X but not in Y.
func CompareStringLists(X, Y []string) []string {
m := make(map[string]bool)
for _, y := range Y {
m[y] = true
}
diff := []string{}
for _, x := range X {
if m[x] {
continue
}
diff = append(diff, x)
m[x] = true
}
return diff
}
// CompareStringListsInBoth returns the strings that are present in both X and Y.
func CompareStringListsInBoth(X, Y []string) []string {
m := make(map[string]struct{})
for _, y := range Y {
m[y] = struct{}{}
}
diff := []string{}
for _, x := range X {
if _, e := m[x]; e {
diff = append(diff, x)
delete(m, x)
}
}
return diff
}

@ -0,0 +1,181 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"archive/tar"
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"errors"
"io"
"io/ioutil"
"os/exec"
"strings"
)
var (
// ErrCouldNotExtract occurs when an extraction fails.
ErrCouldNotExtract = errors.New("utils: could not extract the archive")
// ErrExtractedFileTooBig occurs when a file to extract is too big.
ErrExtractedFileTooBig = errors.New("utils: could not extract one or more files from the archive: file too big")
readLen = 6 // max bytes to sniff
gzipHeader = []byte{0x1f, 0x8b}
bzip2Header = []byte{0x42, 0x5a, 0x68}
xzHeader = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00}
)
// XzReader is an io.ReadCloser which decompresses xz compressed data.
type XzReader struct {
io.ReadCloser
cmd *exec.Cmd
closech chan error
}
// NewXzReader shells out to a command line xz executable (if
// available) to decompress the given io.Reader using the xz
// compression format and returns an *XzReader.
// It is the caller's responsibility to call Close on the XzReader when done.
func NewXzReader(r io.Reader) (*XzReader, error) {
rpipe, wpipe := io.Pipe()
ex, err := exec.LookPath("xz")
if err != nil {
return nil, err
}
cmd := exec.Command(ex, "--decompress", "--stdout")
closech := make(chan error)
cmd.Stdin = r
cmd.Stdout = wpipe
go func() {
err := cmd.Run()
wpipe.CloseWithError(err)
closech <- err
}()
return &XzReader{rpipe, cmd, closech}, nil
}
func (r *XzReader) Close() error {
r.ReadCloser.Close()
r.cmd.Process.Kill()
return <-r.closech
}
// TarReadCloser embeds a *tar.Reader and the related io.Closer
// It is the caller's responsibility to call Close on TarReadCloser when
// done.
type TarReadCloser struct {
*tar.Reader
io.Closer
}
func (r *TarReadCloser) Close() error {
return r.Closer.Close()
}
// SelectivelyExtractArchive extracts the specified files and folders
// from targz data read from the given reader and store them in a map indexed by file paths
func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
data := make(map[string][]byte)
// Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader
tr, err := getTarReader(r)
if err != nil {
return data, ErrCouldNotExtract
}
defer tr.Close()
// For each element in the archive
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return data, ErrCouldNotExtract
}
// Get element filename
filename := hdr.Name
filename = strings.TrimPrefix(filename, "./")
if prefix != "" {
filename = strings.TrimPrefix(filename, prefix)
}
// Determine if we should extract the element
toBeExtracted := false
for _, s := range toExtract {
if strings.HasPrefix(filename, s) {
toBeExtracted = true
break
}
}
if toBeExtracted {
// File size limit
if maxFileSize > 0 && hdr.Size > maxFileSize {
return data, ErrExtractedFileTooBig
}
// Extract the element
if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink || hdr.Typeflag == tar.TypeReg {
d, _ := ioutil.ReadAll(tr)
data[filename] = d
}
}
}
return data, nil
}
// getTarReader returns a TarReaderCloser associated with the specified io.Reader.
//
// Gzip/Bzip2/XZ detection is done by using the magic numbers:
// Gzip: the first two bytes should be 0x1f and 0x8b. Defined in the RFC1952.
// Bzip2: the first three bytes should be 0x42, 0x5a and 0x68. No RFC.
// XZ: the first three bytes should be 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00. No RFC.
func getTarReader(r io.Reader) (*TarReadCloser, error) {
br := bufio.NewReader(r)
header, err := br.Peek(readLen)
if err == nil {
switch {
case bytes.HasPrefix(header, gzipHeader):
gr, err := gzip.NewReader(br)
if err != nil {
return nil, err
}
return &TarReadCloser{tar.NewReader(gr), gr}, nil
case bytes.HasPrefix(header, bzip2Header):
bzip2r := ioutil.NopCloser(bzip2.NewReader(br))
return &TarReadCloser{tar.NewReader(bzip2r), bzip2r}, nil
case bytes.HasPrefix(header, xzHeader):
xzr, err := NewXzReader(br)
if err != nil {
return nil, err
}
return &TarReadCloser{tar.NewReader(xzr), xzr}, nil
}
}
dr := ioutil.NopCloser(br)
return &TarReadCloser{tar.NewReader(dr), dr}, nil
}

@ -0,0 +1,110 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package types defines useful types that are used in database models.
package types
import (
"database/sql/driver"
"errors"
"fmt"
)
// Priority defines a vulnerability priority
type Priority string
const (
// Unknown is either a security problem that has not been
// assigned to a priority yet or a priority that our system
// did not recognize
Unknown Priority = "Unknown"
// Negligible is technically a security problem, but is
// only theoretical in nature, requires a very special
// situation, has almost no install base, or does no real
// damage. These tend not to get backport from upstreams,
// and will likely not be included in security updates unless
// there is an easy fix and some other issue causes an update.
Negligible Priority = "Negligible"
// Low is a security problem, but is hard to
// exploit due to environment, requires a user-assisted
// attack, a small install base, or does very little damage.
// These tend to be included in security updates only when
// higher priority issues require an update, or if many
// low priority issues have built up.
Low Priority = "Low"
// Medium is a real security problem, and is exploitable
// for many people. Includes network daemon denial of service
// attacks, cross-site scripting, and gaining user privileges.
// Updates should be made soon for this priority of issue.
Medium Priority = "Medium"
// High is a real problem, exploitable for many people in a default
// installation. Includes serious remote denial of services,
// local root privilege escalations, or data loss.
High Priority = "High"
// Critical is a world-burning problem, exploitable for nearly all people
// in a default installation of Linux. Includes remote root
// privilege escalations, or massive data loss.
Critical Priority = "Critical"
// Defcon1 is a Critical problem which has been manually highlighted by
// the team. It requires an immediate attention.
Defcon1 Priority = "Defcon1"
)
// Priorities lists all known priorities, ordered from lower to higher
var Priorities = []Priority{Unknown, Negligible, Low, Medium, High, Critical, Defcon1}
// IsValid determines if the priority is a valid one
func (p Priority) IsValid() bool {
for _, pp := range Priorities {
if p == pp {
return true
}
}
return false
}
// Compare compares two priorities
func (p Priority) Compare(p2 Priority) int {
var i1, i2 int
for i1 = 0; i1 < len(Priorities); i1 = i1 + 1 {
if p == Priorities[i1] {
break
}
}
for i2 = 0; i2 < len(Priorities); i2 = i2 + 1 {
if p2 == Priorities[i2] {
break
}
}
return i1 - i2
}
func (p *Priority) Scan(value interface{}) error {
val, ok := value.([]byte)
if !ok {
return errors.New("could not scan a Priority from a non-string input")
}
*p = Priority(string(val))
if !p.IsValid() {
return fmt.Errorf("could not scan an invalid Priority (%v)", p)
}
return nil
}
func (p *Priority) Value() (driver.Value, error) {
return string(*p), nil
}

@ -0,0 +1,296 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"database/sql/driver"
"encoding/json"
"errors"
"strconv"
"strings"
"unicode"
)
// Version represents a package version
type Version struct {
epoch int
version string
revision string
}
var (
// MinVersion is a special package version which is always sorted first
MinVersion = Version{version: "#MINV#"}
// MaxVersion is a special package version which is always sorted last
MaxVersion = Version{version: "#MAXV#"}
versionAllowedSymbols = []rune{'.', '-', '+', '~', ':', '_'}
revisionAllowedSymbols = []rune{'.', '+', '~', '_'}
)
// NewVersion function parses a string into a Version struct which can be compared
//
// The implementation is based on http://man.he.net/man5/deb-version
// on https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
//
// It uses the dpkg-1.17.25's algorithm (lib/parsehelp.c)
func NewVersion(str string) (Version, error) {
var version Version
// Trim leading and trailing space
str = strings.TrimSpace(str)
if len(str) == 0 {
return Version{}, errors.New("Version string is empty")
}
// Max/Min versions
if str == MaxVersion.String() {
return MaxVersion, nil
}
if str == MinVersion.String() {
return MinVersion, nil
}
// Find epoch
sepepoch := strings.Index(str, ":")
if sepepoch > -1 {
intepoch, err := strconv.Atoi(str[:sepepoch])
if err == nil {
version.epoch = intepoch
} else {
return Version{}, errors.New("epoch in version is not a number")
}
if intepoch < 0 {
return Version{}, errors.New("epoch in version is negative")
}
} else {
version.epoch = 0
}
// Find version / revision
seprevision := strings.LastIndex(str, "-")
if seprevision > -1 {
version.version = str[sepepoch+1 : seprevision]
version.revision = str[seprevision+1:]
} else {
version.version = str[sepepoch+1:]
version.revision = ""
}
// Verify format
if len(version.version) == 0 {
return Version{}, errors.New("No version")
}
if !unicode.IsDigit(rune(version.version[0])) {
return Version{}, errors.New("version does not start with digit")
}
for i := 0; i < len(version.version); i = i + 1 {
r := rune(version.version[i])
if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(versionAllowedSymbols, r) {
return Version{}, errors.New("invalid character in version")
}
}
for i := 0; i < len(version.revision); i = i + 1 {
r := rune(version.revision[i])
if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(revisionAllowedSymbols, r) {
return Version{}, errors.New("invalid character in revision")
}
}
return version, nil
}
// NewVersionUnsafe is just a wrapper around NewVersion that ignore potentiel
// parsing error. Useful for test purposes
func NewVersionUnsafe(str string) Version {
v, _ := NewVersion(str)
return v
}
// Compare function compares two Debian-like package version
//
// The implementation is based on http://man.he.net/man5/deb-version
// on https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
//
// It uses the dpkg-1.17.25's algorithm (lib/version.c)
func (a Version) Compare(b Version) int {
// Quick check
if a == b {
return 0
}
// Max/Min comparison
if a == MinVersion || b == MaxVersion {
return -1
}
if b == MinVersion || a == MaxVersion {
return 1
}
// Compare epochs
if a.epoch > b.epoch {
return 1
}
if a.epoch < b.epoch {
return -1
}
// Compare version
rc := verrevcmp(a.version, b.version)
if rc != 0 {
return signum(rc)
}
// Compare revision
return signum(verrevcmp(a.revision, b.revision))
}
// String returns the string representation of a Version
func (v Version) String() (s string) {
if v.epoch != 0 {
s = strconv.Itoa(v.epoch) + ":"
}
s += v.version
if v.revision != "" {
s += "-" + v.revision
}
return
}
func (v Version) MarshalJSON() ([]byte, error) {
return json.Marshal(v.String())
}
func (v *Version) UnmarshalJSON(b []byte) (err error) {
var str string
json.Unmarshal(b, &str)
vp := NewVersionUnsafe(str)
*v = vp
return
}
func (v *Version) Scan(value interface{}) (err error) {
val, ok := value.([]byte)
if !ok {
return errors.New("could not scan a Version from a non-string input")
}
*v, err = NewVersion(string(val))
return
}
func (v *Version) Value() (driver.Value, error) {
return v.String(), nil
}
func verrevcmp(t1, t2 string) int {
t1, rt1 := nextRune(t1)
t2, rt2 := nextRune(t2)
for rt1 != nil || rt2 != nil {
firstDiff := 0
for (rt1 != nil && !unicode.IsDigit(*rt1)) || (rt2 != nil && !unicode.IsDigit(*rt2)) {
ac := 0
bc := 0
if rt1 != nil {
ac = order(*rt1)
}
if rt2 != nil {
bc = order(*rt2)
}
if ac != bc {
return ac - bc
}
t1, rt1 = nextRune(t1)
t2, rt2 = nextRune(t2)
}
for rt1 != nil && *rt1 == '0' {
t1, rt1 = nextRune(t1)
}
for rt2 != nil && *rt2 == '0' {
t2, rt2 = nextRune(t2)
}
for rt1 != nil && unicode.IsDigit(*rt1) && rt2 != nil && unicode.IsDigit(*rt2) {
if firstDiff == 0 {
firstDiff = int(*rt1) - int(*rt2)
}
t1, rt1 = nextRune(t1)
t2, rt2 = nextRune(t2)
}
if rt1 != nil && unicode.IsDigit(*rt1) {
return 1
}
if rt2 != nil && unicode.IsDigit(*rt2) {
return -1
}
if firstDiff != 0 {
return firstDiff
}
}
return 0
}
// order compares runes using a modified ASCII table
// so that letters are sorted earlier than non-letters
// and so that tildes sorts before anything
func order(r rune) int {
if unicode.IsDigit(r) {
return 0
}
if unicode.IsLetter(r) {
return int(r)
}
if r == '~' {
return -1
}
return int(r) + 256
}
func nextRune(str string) (string, *rune) {
if len(str) >= 1 {
r := rune(str[0])
return str[1:], &r
}
return str, nil
}
func containsRune(s []rune, e rune) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func signum(a int) int {
switch {
case a < 0:
return -1
case a > 0:
return +1
}
return 0
}

@ -0,0 +1,105 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package detectors exposes functions to register and use container
// information extractors.
package detectors
import (
"fmt"
"io"
"math"
"net/http"
"os"
"strings"
"sync"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/pkg/capnslog"
)
// The DataDetector interface defines a way to detect the required data from input path
type DataDetector interface {
//Support check if the input path and format are supported by the underling detector
Supported(path string, format string) bool
// Detect detects the required data from input path
Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error)
}
var (
dataDetectorsLock sync.Mutex
dataDetectors = make(map[string]DataDetector)
log = capnslog.NewPackageLogger("github.com/coreos/clair", "detectors")
// ErrCouldNotFindLayer is returned when we could not download or open the layer file.
ErrCouldNotFindLayer = cerrors.NewBadRequestError("could not find layer")
)
// RegisterDataDetector provides a way to dynamically register an implementation of a
// DataDetector.
//
// If RegisterDataDetector is called twice with the same name if DataDetector is nil,
// or if the name is blank, it panics.
func RegisterDataDetector(name string, f DataDetector) {
if name == "" {
panic("Could not register a DataDetector with an empty name")
}
if f == nil {
panic("Could not register a nil DataDetector")
}
dataDetectorsLock.Lock()
defer dataDetectorsLock.Unlock()
if _, alreadyExists := dataDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
dataDetectors[name] = f
}
// DetectData finds the Data of the layer by using every registered DataDetector
func DetectData(path string, format string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) {
var layerReader io.ReadCloser
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
r, err := http.Get(path)
if err != nil {
log.Warningf("could not download layer: %s", err)
return nil, ErrCouldNotFindLayer
}
if math.Floor(float64(r.StatusCode/100)) != 2 {
log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode)
return nil, ErrCouldNotFindLayer
}
layerReader = r.Body
} else {
layerReader, err = os.Open(path)
if err != nil {
return nil, ErrCouldNotFindLayer
}
}
defer layerReader.Close()
for _, detector := range dataDetectors {
if detector.Supported(path, format) {
data, err = detector.Detect(layerReader, toExtract, maxFileSize)
if err != nil {
return nil, err
}
return data, nil
}
}
return nil, cerrors.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format))
}

@ -0,0 +1,41 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aci
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// ACIDataDetector implements DataDetector and detects layer data in 'aci' format
type ACIDataDetector struct{}
func init() {
detectors.RegisterDataDetector("aci", &ACIDataDetector{})
}
func (detector *ACIDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "ACI") {
return true
}
return false
}
func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize)
}

@ -0,0 +1,41 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"io"
"strings"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/worker/detectors"
)
// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format
type DockerDataDetector struct{}
func init() {
detectors.RegisterDataDetector("Docker", &DockerDataDetector{})
}
func (detector *DockerDataDetector) Supported(path string, format string) bool {
if strings.EqualFold(format, "Docker") {
return true
}
return false
}
func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) {
return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize)
}

@ -0,0 +1,115 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dpkg
import (
"bufio"
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
"github.com/coreos/pkg/capnslog"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages")
dpkgSrcCaptureRegexp = regexp.MustCompile(`Source: (?P<name>[^\s]*)( \((?P<version>.*)\))?`)
dpkgSrcCaptureRegexpNames = dpkgSrcCaptureRegexp.SubexpNames()
)
// DpkgFeaturesDetector implements FeaturesDetector and detects dpkg packages
type DpkgFeaturesDetector struct{}
func init() {
detectors.RegisterFeaturesDetector("dpkg", &DpkgFeaturesDetector{})
}
// Detect detects packages using var/lib/dpkg/status from the input data
func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/dpkg/status"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
var pkg database.FeatureVersion
var err error
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
// Package line
// Defines the name of the package
pkg.Feature.Name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
pkg.Version = types.Version{}
} else if strings.HasPrefix(line, "Source: ") {
// Source line (Optionnal)
// Gives the name of the source package
// May also specifies a version
srcCapture := dpkgSrcCaptureRegexp.FindAllStringSubmatch(line, -1)[0]
md := map[string]string{}
for i, n := range srcCapture {
md[dpkgSrcCaptureRegexpNames[i]] = strings.TrimSpace(n)
}
pkg.Feature.Name = md["name"]
if md["version"] != "" {
pkg.Version, err = types.NewVersion(md["version"])
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
}
}
} else if strings.HasPrefix(line, "Version: ") && pkg.Version.String() == "" {
// Version line
// Defines the version of the package
// This version is less important than a version retrieved from a Source line
// because the Debian vulnerabilities often skips the epoch from the Version field
// which is not present in the Source version, and because +bX revisions don't matter
pkg.Version, err = types.NewVersion(strings.TrimPrefix(line, "Version: "))
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
}
}
// Add the package to the result array if we have all the informations
if pkg.Feature.Name != "" && pkg.Version.String() != "" {
packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg
pkg.Feature.Name = ""
pkg.Version = types.Version{}
}
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}
return packages, nil
}
// GetRequiredFiles returns the list of files required for Detect, without
// leading /
func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/dpkg/status"}
}

@ -0,0 +1,120 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rpm
import (
"bufio"
"io/ioutil"
"os"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
"github.com/coreos/pkg/capnslog"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "rpm")
// RpmFeaturesDetector implements FeaturesDetector and detects rpm packages
// It requires the "rpm" binary to be in the PATH
type RpmFeaturesDetector struct{}
func init() {
detectors.RegisterFeaturesDetector("rpm", &RpmFeaturesDetector{})
}
// Detect detects packages using var/lib/rpm/Packages from the input data
func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
f, hasFile := data["var/lib/rpm/Packages"]
if !hasFile {
return []database.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
// Write the required "Packages" file to disk
tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm")
defer os.RemoveAll(tmpDir)
if err != nil {
log.Errorf("could not create temporary folder for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
}
err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700)
if err != nil {
log.Errorf("could not create temporary file for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
}
// Query RPM
// We actually extract binary package names instead of source package names here because RHSA refers to package names
// In the dpkg system, we extract the source instead
out, err := utils.Exec(tmpDir, "rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n")
if err != nil {
log.Errorf("could not query RPM: %s. output: %s", err, string(out))
// Do not bubble up because we probably won't be able to fix it,
// the database must be corrupted
return []database.FeatureVersion{}, nil
}
scanner := bufio.NewScanner(strings.NewReader(string(out)))
for scanner.Scan() {
line := strings.Split(scanner.Text(), " ")
if len(line) != 2 {
// We may see warnings on some RPM versions:
// "warning: Generating 12 missing index(es), please wait..."
continue
}
// Ignore gpg-pubkey packages which are fake packages used to store GPG keys - they are not versionned properly.
if line[0] == "gpg-pubkey" {
continue
}
// Parse version
version, err := types.NewVersion(strings.Replace(line[1], "(none):", "", -1))
if err != nil {
log.Warningf("could not parse package version '%s': %s. skipping", line[1], err.Error())
continue
}
// Add package
pkg := database.FeatureVersion{
Feature: database.Feature{
Name: line[0],
},
Version: version,
}
packagesMap[pkg.Feature.Name+"#"+pkg.Version.String()] = pkg
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}
return packages, nil
}
// GetRequiredFiles returns the list of files required for Detect, without
// leading /
func (detector *RpmFeaturesDetector) GetRequiredFiles() []string {
return []string{"var/lib/rpm/Packages"}
}

@ -0,0 +1,48 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package feature
import (
"io/ioutil"
"path"
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
"github.com/stretchr/testify/assert"
)
type FeatureVersionTest struct {
FeatureVersions []database.FeatureVersion
Data map[string][]byte
}
func LoadFileForTest(name string) []byte {
_, filename, _, _ := runtime.Caller(0)
d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/" + name)
return d
}
func TestFeaturesDetector(t *testing.T, detector detectors.FeaturesDetector, tests []FeatureVersionTest) {
for _, test := range tests {
featureVersions, err := detector.Detect(test.Data)
if assert.Nil(t, err) && assert.Len(t, featureVersions, len(test.FeatureVersions)) {
for _, expectedFeatureVersion := range test.FeatureVersions {
assert.Contains(t, featureVersions, expectedFeatureVersion)
}
}
}
}

@ -0,0 +1,79 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package detectors
import (
"fmt"
"sync"
"github.com/coreos/clair/database"
)
// The FeaturesDetector interface defines a way to detect packages from input data.
type FeaturesDetector interface {
// Detect detects a list of FeatureVersion from the input data.
Detect(map[string][]byte) ([]database.FeatureVersion, error)
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
}
var (
featuresDetectorsLock sync.Mutex
featuresDetectors = make(map[string]FeaturesDetector)
)
// RegisterFeaturesDetector makes a FeaturesDetector available for DetectFeatures.
func RegisterFeaturesDetector(name string, f FeaturesDetector) {
if name == "" {
panic("Could not register a FeaturesDetector with an empty name")
}
if f == nil {
panic("Could not register a nil FeaturesDetector")
}
featuresDetectorsLock.Lock()
defer featuresDetectorsLock.Unlock()
if _, alreadyExists := featuresDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
featuresDetectors[name] = f
}
// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector.
func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) {
var packages []database.FeatureVersion
for _, detector := range featuresDetectors {
pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
}
packages = append(packages, pkgs...)
}
return packages, nil
}
// GetRequiredFilesFeatures returns the list of files required for Detect for every
// registered FeaturesDetector, without leading /.
func GetRequiredFilesFeatures() (files []string) {
for _, detector := range featuresDetectors {
files = append(files, detector.GetRequiredFiles()...)
}
return
}

@ -0,0 +1,82 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package detectors exposes functions to register and use container
// information extractors.
package detectors
import (
"fmt"
"sync"
"github.com/coreos/clair/database"
)
// The NamespaceDetector interface defines a way to detect a Namespace from input data.
// A namespace is usually made of an Operating System name and its version.
type NamespaceDetector interface {
// Detect detects a Namespace and its version from input data.
Detect(map[string][]byte) *database.Namespace
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
}
var (
namespaceDetectorsLock sync.Mutex
namespaceDetectors = make(map[string]NamespaceDetector)
)
// RegisterNamespaceDetector provides a way to dynamically register an implementation of a
// NamespaceDetector.
//
// If RegisterNamespaceDetector is called twice with the same name if NamespaceDetector is nil,
// or if the name is blank, it panics.
func RegisterNamespaceDetector(name string, f NamespaceDetector) {
if name == "" {
panic("Could not register a NamespaceDetector with an empty name")
}
if f == nil {
panic("Could not register a nil NamespaceDetector")
}
namespaceDetectorsLock.Lock()
defer namespaceDetectorsLock.Unlock()
if _, alreadyExists := namespaceDetectors[name]; alreadyExists {
panic(fmt.Sprintf("Detector '%s' is already registered", name))
}
namespaceDetectors[name] = f
}
// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector.
func DetectNamespace(data map[string][]byte) *database.Namespace {
for _, detector := range namespaceDetectors {
if namespace := detector.Detect(data); namespace != nil {
return namespace
}
}
return nil
}
// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every
// registered NamespaceDetector, without leading /.
func GetRequiredFilesNamespace() (files []string) {
for _, detector := range namespaceDetectors {
files = append(files, detector.GetRequiredFiles()...)
}
return
}

@ -0,0 +1,85 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aptsources
import (
"bufio"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
)
// AptSourcesNamespaceDetector implements NamespaceDetector and detects the Namespace from the
// /etc/apt/sources.list file.
//
// This detector is necessary to determine precise Debian version when it is
// an unstable version for instance.
type AptSourcesNamespaceDetector struct{}
func init() {
detectors.RegisterNamespaceDetector("apt-sources", &AptSourcesNamespaceDetector{})
}
func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
f, hasFile := data["etc/apt/sources.list"]
if !hasFile {
return nil
}
var OS, version string
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
// Format: man sources.list | https://wiki.debian.org/SourcesList)
// deb uri distribution component1 component2 component3
// deb-src uri distribution component1 component2 component3
line := strings.Split(scanner.Text(), " ")
if len(line) > 3 {
// Only consider main component
isMainComponent := false
for _, component := range line[3:] {
if component == "main" {
isMainComponent = true
break
}
}
if !isMainComponent {
continue
}
var found bool
version, found = database.DebianReleasesMapping[line[2]]
if found {
OS = "debian"
break
}
version, found = database.UbuntuReleasesMapping[line[2]]
if found {
OS = "ubuntu"
break
}
}
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
}
return nil
}
func (detector *AptSourcesNamespaceDetector) GetRequiredFiles() []string {
return []string{"etc/apt/sources.list"}
}

@ -0,0 +1,81 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lsbrelease
import (
"bufio"
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
)
var (
lsbReleaseOSRegexp = regexp.MustCompile(`^DISTRIB_ID=(.*)`)
lsbReleaseVersionRegexp = regexp.MustCompile(`^DISTRIB_RELEASE=(.*)`)
)
// AptSourcesNamespaceDetector implements NamespaceDetector and detects the Namespace from the
// /etc/lsb-release file.
//
// This detector is necessary for Ubuntu Precise.
type LsbReleaseNamespaceDetector struct{}
func init() {
detectors.RegisterNamespaceDetector("lsb-release", &LsbReleaseNamespaceDetector{})
}
func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
f, hasFile := data["etc/lsb-release"]
if !hasFile {
return nil
}
var OS, version string
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
r := lsbReleaseOSRegexp.FindStringSubmatch(line)
if len(r) == 2 {
OS = strings.Replace(strings.ToLower(r[1]), "\"", "", -1)
}
r = lsbReleaseVersionRegexp.FindStringSubmatch(line)
if len(r) == 2 {
version = strings.Replace(strings.ToLower(r[1]), "\"", "", -1)
// We care about the .04 for Ubuntu but not for Debian / CentOS
if OS == "centos" || OS == "debian" {
i := strings.Index(version, ".")
if i >= 0 {
version = version[:i]
}
}
}
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
}
return nil
}
// GetRequiredFiles returns the list of files that are required for Detect()
func (detector *LsbReleaseNamespaceDetector) GetRequiredFiles() []string {
return []string{"etc/lsb-release"}
}

@ -0,0 +1,76 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package osrelease
import (
"bufio"
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
)
var (
osReleaseOSRegexp = regexp.MustCompile(`^ID=(.*)`)
osReleaseVersionRegexp = regexp.MustCompile(`^VERSION_ID=(.*)`)
)
// OsReleaseNamespaceDetector implements NamespaceDetector and detects the OS from the
// /etc/os-release and usr/lib/os-release files.
type OsReleaseNamespaceDetector struct{}
func init() {
detectors.RegisterNamespaceDetector("os-release", &OsReleaseNamespaceDetector{})
}
// Detect tries to detect OS/Version using "/etc/os-release" and "/usr/lib/os-release"
// Typically for Debian / Ubuntu
// /etc/debian_version can't be used, it does not make any difference between testing and unstable, it returns stretch/sid
func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
var OS, version string
for _, filePath := range detector.GetRequiredFiles() {
f, hasFile := data[filePath]
if !hasFile {
continue
}
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
line := scanner.Text()
r := osReleaseOSRegexp.FindStringSubmatch(line)
if len(r) == 2 {
OS = strings.Replace(strings.ToLower(r[1]), "\"", "", -1)
}
r = osReleaseVersionRegexp.FindStringSubmatch(line)
if len(r) == 2 {
version = strings.Replace(strings.ToLower(r[1]), "\"", "", -1)
}
}
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
}
return nil
}
// GetRequiredFiles returns the list of files that are required for Detect()
func (detector *OsReleaseNamespaceDetector) GetRequiredFiles() []string {
return []string{"etc/os-release", "usr/lib/os-release"}
}

@ -0,0 +1,59 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package redhatrelease
import (
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
)
var redhatReleaseRegexp = regexp.MustCompile(`(?P<os>[^\s]*) (Linux release|release) (?P<version>[\d]+)`)
// RedhatReleaseNamespaceDetector implements NamespaceDetector and detects the OS from the
// /etc/centos-release, /etc/redhat-release and /etc/system-release files.
//
// Typically for CentOS and Red-Hat like systems
// eg. CentOS release 5.11 (Final)
// eg. CentOS release 6.6 (Final)
// eg. CentOS Linux release 7.1.1503 (Core)
type RedhatReleaseNamespaceDetector struct{}
func init() {
detectors.RegisterNamespaceDetector("redhat-release", &RedhatReleaseNamespaceDetector{})
}
func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
for _, filePath := range detector.GetRequiredFiles() {
f, hasFile := data[filePath]
if !hasFile {
continue
}
r := redhatReleaseRegexp.FindStringSubmatch(string(f))
if len(r) == 4 {
return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]}
}
}
return nil
}
// GetRequiredFiles returns the list of files that are required for Detect()
func (detector *RedhatReleaseNamespaceDetector) GetRequiredFiles() []string {
return []string{"etc/centos-release", "etc/redhat-release", "etc/system-release"}
}

@ -0,0 +1,34 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package namespace
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/worker/detectors"
"github.com/stretchr/testify/assert"
)
type NamespaceTest struct {
Data map[string][]byte
ExpectedNamespace database.Namespace
}
func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) {
for _, test := range tests {
assert.Equal(t, test.ExpectedNamespace, *detector.Detect(test.Data))
}
}

@ -0,0 +1,192 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package worker implements the logic to extract useful informations from a
// container layer and store it in the database.
package worker
import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker/detectors"
)
const (
// Version (integer) represents the worker version.
// Increased each time the engine changes.
Version = 2
// maxFileSize is the maximum size of a single file we should extract.
maxFileSize = 200 * 1024 * 1024 // 200 MiB
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker")
// ErrUnsupported is the error that should be raised when an OS or package
// manager is not supported.
ErrUnsupported = cerrors.NewBadRequestError("worker: OS and/or package manager are not supported")
// ErrParentUnknown is the error that should be raised when a parent layer
// has yet to be processed for the current layer.
ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
)
// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, name, parentName, path, imageFormat string) error {
// Verify parameters.
if name == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a name")
}
if path == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a path")
}
if imageFormat == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a format")
}
log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
name, utils.CleanURL(path), Version, parentName, imageFormat)
// Check to see if the layer is already in the database.
layer, err := datastore.FindLayer(name, false, false)
if err != nil && err != cerrors.ErrNotFound {
return err
}
if err == cerrors.ErrNotFound {
// New layer case.
layer = database.Layer{Name: name, EngineVersion: Version}
// Retrieve the parent if it has one.
// We need to get it with its Features in order to diff them.
if parentName != "" {
parent, err := datastore.FindLayer(parentName, true, false)
if err != nil && err != cerrors.ErrNotFound {
return err
}
if err == cerrors.ErrNotFound {
log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
parentName)
return ErrParentUnknown
}
layer.Parent = &parent
}
} else {
// The layer is already in the database, check if we need to update it.
if layer.EngineVersion >= Version {
log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
return nil
}
log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
engine is %d. analyzing again`, name, layer.EngineVersion, Version)
}
// Analyze the content.
layer.Namespace, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent)
if err != nil {
return err
}
return datastore.InsertLayer(layer)
}
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(name, path, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
data, err := detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesFeatures(),
detectors.GetRequiredFilesNamespace()...), maxFileSize)
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
return
}
// Detect namespace.
namespace, err = detectNamespace(data, parent)
if err != nil {
return
}
if namespace != nil {
log.Debugf("layer %s: Namespace is %s.", name, namespace.Name)
} else {
log.Debugf("layer %s: OS is unknown.", name)
}
// Detect features.
features, err = detectFeatures(name, data, namespace)
if err != nil {
return
}
// If there are no feature detected, use parent's features if possible.
// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some
// parent's Features. It would be useful for detectors that can't find their entire result using
// one Layer.
if len(features) == 0 && parent != nil {
features = parent.Features
}
log.Debugf("layer %s: detected %d features", name, len(features))
return
}
func detectNamespace(data map[string][]byte, parent *database.Layer) (namespace *database.Namespace, err error) {
namespace = detectors.DetectNamespace(data)
// Attempt to detect the OS from the parent layer.
if namespace == nil && parent != nil {
namespace = parent.Namespace
if err != nil {
return
}
}
return
}
func detectFeatures(name string, data map[string][]byte, namespace *database.Namespace) (features []database.FeatureVersion, err error) {
// TODO(Quentin-M): We need to pass the parent image DetectFeatures because it's possible that
// some detectors would need it in order to produce the entire feature list (if they can only
// detect a diff). Also, we should probably pass the detected namespace so detectors could
// make their own decision.
features, err = detectors.DetectFeatures(data)
if err != nil {
return
}
// Ensure that every feature has a Namespace associated, otherwise associate the detected
// namespace. If there is no detected namespace, we'll throw an error.
for i := 0; i < len(features); i++ {
if features[i].Feature.Namespace.Name == "" {
if namespace != nil {
features[i].Feature.Namespace = *namespace
} else {
log.Warningf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
err = ErrUnsupported
return
}
}
}
return
}

@ -0,0 +1,191 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:
You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.
8. Limitation of Liability.
In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@ -0,0 +1,178 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package journal provides write bindings to the local systemd journal.
// It is implemented in pure Go and connects to the journal directly over its
// unix socket.
//
// To read from the journal, see the "sdjournal" package, which wraps the
// sd-journal a C API.
//
// http://www.freedesktop.org/software/systemd/man/systemd-journald.service.html
package journal
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"strconv"
"strings"
"syscall"
)
// Priority of a journal message
type Priority int
const (
PriEmerg Priority = iota
PriAlert
PriCrit
PriErr
PriWarning
PriNotice
PriInfo
PriDebug
)
var conn net.Conn
func init() {
var err error
conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
if err != nil {
conn = nil
}
}
// Enabled returns true if the local systemd journal is available for logging
func Enabled() bool {
return conn != nil
}
// Send a message to the local systemd journal. vars is a map of journald
// fields to values. Fields must be composed of uppercase letters, numbers,
// and underscores, but must not start with an underscore. Within these
// restrictions, any arbitrary field name may be used. Some names have special
// significance: see the journalctl documentation
// (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html)
// for more details. vars may be nil.
func Send(message string, priority Priority, vars map[string]string) error {
if conn == nil {
return journalError("could not connect to journald socket")
}
data := new(bytes.Buffer)
appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
appendVariable(data, "MESSAGE", message)
for k, v := range vars {
appendVariable(data, k, v)
}
_, err := io.Copy(conn, data)
if err != nil && isSocketSpaceError(err) {
file, err := tempFd()
if err != nil {
return journalError(err.Error())
}
_, err = io.Copy(file, data)
if err != nil {
return journalError(err.Error())
}
rights := syscall.UnixRights(int(file.Fd()))
/* this connection should always be a UnixConn, but better safe than sorry */
unixConn, ok := conn.(*net.UnixConn)
if !ok {
return journalError("can't send file through non-Unix connection")
}
unixConn.WriteMsgUnix([]byte{}, rights, nil)
} else if err != nil {
return journalError(err.Error())
}
return nil
}
// Print prints a message to the local systemd journal using Send().
func Print(priority Priority, format string, a ...interface{}) error {
return Send(fmt.Sprintf(format, a...), priority, nil)
}
func appendVariable(w io.Writer, name, value string) {
if !validVarName(name) {
journalError("variable name contains invalid character, ignoring")
}
if strings.ContainsRune(value, '\n') {
/* When the value contains a newline, we write:
* - the variable name, followed by a newline
* - the size (in 64bit little endian format)
* - the data, followed by a newline
*/
fmt.Fprintln(w, name)
binary.Write(w, binary.LittleEndian, uint64(len(value)))
fmt.Fprintln(w, value)
} else {
/* just write the variable and value all on one line */
fmt.Fprintf(w, "%s=%s\n", name, value)
}
}
func validVarName(name string) bool {
/* The variable name must be in uppercase and consist only of characters,
* numbers and underscores, and may not begin with an underscore. (from the docs)
*/
valid := name[0] != '_'
for _, c := range name {
valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_'
}
return valid
}
func isSocketSpaceError(err error) bool {
opErr, ok := err.(*net.OpError)
if !ok {
return false
}
sysErr, ok := opErr.Err.(syscall.Errno)
if !ok {
return false
}
return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}
func tempFd() (*os.File, error) {
file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
if err != nil {
return nil, err
}
syscall.Unlink(file.Name())
if err != nil {
return nil, err
}
return file, nil
}
func journalError(s string) error {
s = "journal error: " + s
fmt.Fprintln(os.Stderr, s)
return errors.New(s)
}

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

@ -0,0 +1,5 @@
CoreOS Project
Copyright 2014 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).

@ -0,0 +1,39 @@
# capnslog, the CoreOS logging package
There are far too many logging packages out there, with varying degrees of licenses, far too many features (colorization, all sorts of log frameworks) or are just a pain to use (lack of `Fatalln()`?).
capnslog provides a simple but consistent logging interface suitable for all kinds of projects.
### Design Principles
##### `package main` is the place where logging gets turned on and routed
A library should not touch log options, only generate log entries. Libraries are silent until main lets them speak.
##### All log options are runtime-configurable.
Still the job of `main` to expose these configurations. `main` may delegate this to, say, a configuration webhook, but does so explicitly.
##### There is one log object per package. It is registered under its repository and package name.
`main` activates logging for its repository and any dependency repositories it would also like to have output in its logstream. `main` also dictates at which level each subpackage logs.
##### There is *one* output stream, and it is an `io.Writer` composed with a formatter.
Splitting streams is probably not the job of your program, but rather, your log aggregation framework. If you must split output streams, again, `main` configures this and you can write a very simple two-output struct that satisfies io.Writer.
Fancy colorful formatting and JSON output are beyond the scope of a basic logging framework -- they're application/log-collector dependant. These are, at best, provided as options, but more likely, provided by your application.
##### Log objects are an interface
An object knows best how to print itself. Log objects can collect more interesting metadata if they wish, however, because text isn't going away anytime soon, they must all be marshalable to text. The simplest log object is a string, which returns itself. If you wish to do more fancy tricks for printing your log objects, see also JSON output -- introspect and write a formatter which can handle your advanced log interface. Making strings is the only thing guaranteed.
##### Log levels have specific meanings:
* Critical: Unrecoverable. Must fail.
* Error: Data has been lost, a request has failed for a bad reason, or a required resource has been lost
* Warning: (Hopefully) Temporary conditions that may cause errors, but may work fine. A replica disappearing (that may reconnect) is a warning.
* Notice: Normal, but important (uncommon) log information.
* Info: Normal, working log information, everything is fine, but helpful notices for auditing or common operations.
* Debug: Everything is still fine, but even common operations may be logged, and less helpful but more quantity of notices.
* Trace: Anything goes, from logging every function call as part of a common operation, to tracing execution of a query.

@ -0,0 +1,57 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
oldlog "log"
"github.com/coreos/pkg/capnslog"
)
var logLevel = capnslog.INFO
var log = capnslog.NewPackageLogger("github.com/coreos/pkg/capnslog/cmd", "main")
var dlog = capnslog.NewPackageLogger("github.com/coreos/pkg/capnslog/cmd", "dolly")
func init() {
flag.Var(&logLevel, "log-level", "Global log level.")
}
func main() {
rl := capnslog.MustRepoLogger("github.com/coreos/pkg/capnslog/cmd")
// We can parse the log level configs from the command line
flag.Parse()
if flag.NArg() > 1 {
cfg, err := rl.ParseLogLevelConfig(flag.Arg(1))
if err != nil {
log.Fatal(err)
}
rl.SetLogLevel(cfg)
log.Infof("Setting output to %s", flag.Arg(1))
}
// Send some messages at different levels to the different packages
dlog.Infof("Hello Dolly")
dlog.Warningf("Well hello, Dolly")
log.Errorf("It's so nice to have you back where you belong")
dlog.Debugf("You're looking swell, Dolly")
dlog.Tracef("I can tell, Dolly")
// We also have control over the built-in "log" package.
capnslog.SetGlobalLogLevel(logLevel)
oldlog.Println("You're still glowin', you're still crowin', you're still lookin' strong")
log.Fatalf("Dolly'll never go away again")
}

@ -0,0 +1,106 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import (
"bufio"
"fmt"
"io"
"runtime"
"strings"
"time"
)
type Formatter interface {
Format(pkg string, level LogLevel, depth int, entries ...interface{})
Flush()
}
func NewStringFormatter(w io.Writer) *StringFormatter {
return &StringFormatter{
w: bufio.NewWriter(w),
}
}
type StringFormatter struct {
w *bufio.Writer
}
func (s *StringFormatter) Format(pkg string, l LogLevel, i int, entries ...interface{}) {
now := time.Now().UTC()
s.w.WriteString(now.Format(time.RFC3339))
s.w.WriteByte(' ')
writeEntries(s.w, pkg, l, i, entries...)
s.Flush()
}
func writeEntries(w *bufio.Writer, pkg string, _ LogLevel, _ int, entries ...interface{}) {
if pkg != "" {
w.WriteString(pkg + ": ")
}
str := fmt.Sprint(entries...)
endsInNL := strings.HasSuffix(str, "\n")
w.WriteString(str)
if !endsInNL {
w.WriteString("\n")
}
}
func (s *StringFormatter) Flush() {
s.w.Flush()
}
func NewPrettyFormatter(w io.Writer, debug bool) Formatter {
return &PrettyFormatter{
w: bufio.NewWriter(w),
debug: debug,
}
}
type PrettyFormatter struct {
w *bufio.Writer
debug bool
}
func (c *PrettyFormatter) Format(pkg string, l LogLevel, depth int, entries ...interface{}) {
now := time.Now()
ts := now.Format("2006-01-02 15:04:05")
c.w.WriteString(ts)
ms := now.Nanosecond() / 1000
c.w.WriteString(fmt.Sprintf(".%06d", ms))
if c.debug {
_, file, line, ok := runtime.Caller(depth) // It's always the same number of frames to the user's call.
if !ok {
file = "???"
line = 1
} else {
slash := strings.LastIndex(file, "/")
if slash >= 0 {
file = file[slash+1:]
}
}
if line < 0 {
line = 0 // not a real line number
}
c.w.WriteString(fmt.Sprintf(" [%s:%d]", file, line))
}
c.w.WriteString(fmt.Sprint(" ", l.Char(), " | "))
writeEntries(c.w, pkg, l, depth, entries...)
c.Flush()
}
func (c *PrettyFormatter) Flush() {
c.w.Flush()
}

@ -0,0 +1,96 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import (
"bufio"
"bytes"
"io"
"os"
"runtime"
"strconv"
"strings"
"time"
)
var pid = os.Getpid()
type GlogFormatter struct {
StringFormatter
}
func NewGlogFormatter(w io.Writer) *GlogFormatter {
g := &GlogFormatter{}
g.w = bufio.NewWriter(w)
return g
}
func (g GlogFormatter) Format(pkg string, level LogLevel, depth int, entries ...interface{}) {
g.w.Write(GlogHeader(level, depth+1))
g.StringFormatter.Format(pkg, level, depth+1, entries...)
}
func GlogHeader(level LogLevel, depth int) []byte {
// Lmmdd hh:mm:ss.uuuuuu threadid file:line]
now := time.Now().UTC()
_, file, line, ok := runtime.Caller(depth) // It's always the same number of frames to the user's call.
if !ok {
file = "???"
line = 1
} else {
slash := strings.LastIndex(file, "/")
if slash >= 0 {
file = file[slash+1:]
}
}
if line < 0 {
line = 0 // not a real line number
}
buf := &bytes.Buffer{}
buf.Grow(30)
_, month, day := now.Date()
hour, minute, second := now.Clock()
buf.WriteString(level.Char())
twoDigits(buf, int(month))
twoDigits(buf, day)
buf.WriteByte(' ')
twoDigits(buf, hour)
buf.WriteByte(':')
twoDigits(buf, minute)
buf.WriteByte(':')
twoDigits(buf, second)
buf.WriteByte('.')
buf.WriteString(strconv.Itoa(now.Nanosecond() / 1000))
buf.WriteByte('Z')
buf.WriteByte(' ')
buf.WriteString(strconv.Itoa(pid))
buf.WriteByte(' ')
buf.WriteString(file)
buf.WriteByte(':')
buf.WriteString(strconv.Itoa(line))
buf.WriteByte(']')
buf.WriteByte(' ')
return buf.Bytes()
}
const digits = "0123456789"
func twoDigits(b *bytes.Buffer, d int) {
c2 := digits[d%10]
d /= 10
c1 := digits[d%10]
b.WriteByte(c1)
b.WriteByte(c2)
}

@ -0,0 +1,49 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build !windows
package capnslog
import (
"io"
"os"
"syscall"
)
// Here's where the opinionation comes in. We need some sensible defaults,
// especially after taking over the log package. Your project (whatever it may
// be) may see things differently. That's okay; there should be no defaults in
// the main package that cannot be controlled or overridden programatically,
// otherwise it's a bug. Doing so is creating your own init_log.go file much
// like this one.
func init() {
initHijack()
// Go `log` pacakge uses os.Stderr.
SetFormatter(NewDefaultFormatter(os.Stderr))
SetGlobalLogLevel(INFO)
}
func NewDefaultFormatter(out io.Writer) Formatter {
if syscall.Getppid() == 1 {
// We're running under init, which may be systemd.
f, err := NewJournaldFormatter()
if err == nil {
return f
}
}
return NewPrettyFormatter(out, false)
}

@ -0,0 +1,25 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import "os"
func init() {
initHijack()
// Go `log` package uses os.Stderr.
SetFormatter(NewPrettyFormatter(os.Stderr, false))
SetGlobalLogLevel(INFO)
}

@ -0,0 +1,68 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build !windows
package capnslog
import (
"errors"
"fmt"
"os"
"path/filepath"
"github.com/coreos/go-systemd/journal"
)
func NewJournaldFormatter() (Formatter, error) {
if !journal.Enabled() {
return nil, errors.New("No systemd detected")
}
return &journaldFormatter{}, nil
}
type journaldFormatter struct{}
func (j *journaldFormatter) Format(pkg string, l LogLevel, _ int, entries ...interface{}) {
var pri journal.Priority
switch l {
case CRITICAL:
pri = journal.PriCrit
case ERROR:
pri = journal.PriErr
case WARNING:
pri = journal.PriWarning
case NOTICE:
pri = journal.PriNotice
case INFO:
pri = journal.PriInfo
case DEBUG:
pri = journal.PriDebug
case TRACE:
pri = journal.PriDebug
default:
panic("Unhandled loglevel")
}
msg := fmt.Sprint(entries...)
tags := map[string]string{
"PACKAGE": pkg,
"SYSLOG_IDENTIFIER": filepath.Base(os.Args[0]),
}
err := journal.Send(msg, pri, tags)
if err != nil {
fmt.Fprintln(os.Stderr, err)
}
}
func (j *journaldFormatter) Flush() {}

@ -0,0 +1,39 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import (
"log"
)
func initHijack() {
pkg := NewPackageLogger("log", "")
w := packageWriter{pkg}
log.SetFlags(0)
log.SetPrefix("")
log.SetOutput(w)
}
type packageWriter struct {
pl *PackageLogger
}
func (p packageWriter) Write(b []byte) (int, error) {
if p.pl.level < INFO {
return 0, nil
}
p.pl.internalLog(calldepth+2, INFO, string(b))
return len(b), nil
}

@ -0,0 +1,240 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import (
"errors"
"strings"
"sync"
)
// LogLevel is the set of all log levels.
type LogLevel int8
const (
// CRITICAL is the lowest log level; only errors which will end the program will be propagated.
CRITICAL LogLevel = iota - 1
// ERROR is for errors that are not fatal but lead to troubling behavior.
ERROR
// WARNING is for errors which are not fatal and not errors, but are unusual. Often sourced from misconfigurations.
WARNING
// NOTICE is for normal but significant conditions.
NOTICE
// INFO is a log level for common, everyday log updates.
INFO
// DEBUG is the default hidden level for more verbose updates about internal processes.
DEBUG
// TRACE is for (potentially) call by call tracing of programs.
TRACE
)
// Char returns a single-character representation of the log level.
func (l LogLevel) Char() string {
switch l {
case CRITICAL:
return "C"
case ERROR:
return "E"
case WARNING:
return "W"
case NOTICE:
return "N"
case INFO:
return "I"
case DEBUG:
return "D"
case TRACE:
return "T"
default:
panic("Unhandled loglevel")
}
}
// String returns a multi-character representation of the log level.
func (l LogLevel) String() string {
switch l {
case CRITICAL:
return "CRITICAL"
case ERROR:
return "ERROR"
case WARNING:
return "WARNING"
case NOTICE:
return "NOTICE"
case INFO:
return "INFO"
case DEBUG:
return "DEBUG"
case TRACE:
return "TRACE"
default:
panic("Unhandled loglevel")
}
}
// Update using the given string value. Fulfills the flag.Value interface.
func (l *LogLevel) Set(s string) error {
value, err := ParseLevel(s)
if err != nil {
return err
}
*l = value
return nil
}
// ParseLevel translates some potential loglevel strings into their corresponding levels.
func ParseLevel(s string) (LogLevel, error) {
switch s {
case "CRITICAL", "C":
return CRITICAL, nil
case "ERROR", "0", "E":
return ERROR, nil
case "WARNING", "1", "W":
return WARNING, nil
case "NOTICE", "2", "N":
return NOTICE, nil
case "INFO", "3", "I":
return INFO, nil
case "DEBUG", "4", "D":
return DEBUG, nil
case "TRACE", "5", "T":
return TRACE, nil
}
return CRITICAL, errors.New("couldn't parse log level " + s)
}
type RepoLogger map[string]*PackageLogger
type loggerStruct struct {
sync.Mutex
repoMap map[string]RepoLogger
formatter Formatter
}
// logger is the global logger
var logger = new(loggerStruct)
// SetGlobalLogLevel sets the log level for all packages in all repositories
// registered with capnslog.
func SetGlobalLogLevel(l LogLevel) {
logger.Lock()
defer logger.Unlock()
for _, r := range logger.repoMap {
r.setRepoLogLevelInternal(l)
}
}
// GetRepoLogger may return the handle to the repository's set of packages' loggers.
func GetRepoLogger(repo string) (RepoLogger, error) {
logger.Lock()
defer logger.Unlock()
r, ok := logger.repoMap[repo]
if !ok {
return nil, errors.New("no packages registered for repo " + repo)
}
return r, nil
}
// MustRepoLogger returns the handle to the repository's packages' loggers.
func MustRepoLogger(repo string) RepoLogger {
r, err := GetRepoLogger(repo)
if err != nil {
panic(err)
}
return r
}
// SetRepoLogLevel sets the log level for all packages in the repository.
func (r RepoLogger) SetRepoLogLevel(l LogLevel) {
logger.Lock()
defer logger.Unlock()
r.setRepoLogLevelInternal(l)
}
func (r RepoLogger) setRepoLogLevelInternal(l LogLevel) {
for _, v := range r {
v.level = l
}
}
// ParseLogLevelConfig parses a comma-separated string of "package=loglevel", in
// order, and returns a map of the results, for use in SetLogLevel.
func (r RepoLogger) ParseLogLevelConfig(conf string) (map[string]LogLevel, error) {
setlist := strings.Split(conf, ",")
out := make(map[string]LogLevel)
for _, setstring := range setlist {
setting := strings.Split(setstring, "=")
if len(setting) != 2 {
return nil, errors.New("oddly structured `pkg=level` option: " + setstring)
}
l, err := ParseLevel(setting[1])
if err != nil {
return nil, err
}
out[setting[0]] = l
}
return out, nil
}
// SetLogLevel takes a map of package names within a repository to their desired
// loglevel, and sets the levels appropriately. Unknown packages are ignored.
// "*" is a special package name that corresponds to all packages, and will be
// processed first.
func (r RepoLogger) SetLogLevel(m map[string]LogLevel) {
logger.Lock()
defer logger.Unlock()
if l, ok := m["*"]; ok {
r.setRepoLogLevelInternal(l)
}
for k, v := range m {
l, ok := r[k]
if !ok {
continue
}
l.level = v
}
}
// SetFormatter sets the formatting function for all logs.
func SetFormatter(f Formatter) {
logger.Lock()
defer logger.Unlock()
logger.formatter = f
}
// NewPackageLogger creates a package logger object.
// This should be defined as a global var in your package, referencing your repo.
func NewPackageLogger(repo string, pkg string) (p *PackageLogger) {
logger.Lock()
defer logger.Unlock()
if logger.repoMap == nil {
logger.repoMap = make(map[string]RepoLogger)
}
r, rok := logger.repoMap[repo]
if !rok {
logger.repoMap[repo] = make(RepoLogger)
r = logger.repoMap[repo]
}
p, pok := r[pkg]
if !pok {
r[pkg] = &PackageLogger{
pkg: pkg,
level: INFO,
}
p = r[pkg]
}
return
}

@ -0,0 +1,158 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package capnslog
import (
"fmt"
"os"
)
type PackageLogger struct {
pkg string
level LogLevel
}
const calldepth = 2
func (p *PackageLogger) internalLog(depth int, inLevel LogLevel, entries ...interface{}) {
if inLevel != CRITICAL && p.level < inLevel {
return
}
logger.Lock()
defer logger.Unlock()
if logger.formatter != nil {
logger.formatter.Format(p.pkg, inLevel, depth+1, entries...)
}
}
func (p *PackageLogger) LevelAt(l LogLevel) bool {
return p.level >= l
}
// Log a formatted string at any level between ERROR and TRACE
func (p *PackageLogger) Logf(l LogLevel, format string, args ...interface{}) {
p.internalLog(calldepth, l, fmt.Sprintf(format, args...))
}
// Log a message at any level between ERROR and TRACE
func (p *PackageLogger) Log(l LogLevel, args ...interface{}) {
p.internalLog(calldepth, l, fmt.Sprint(args...))
}
// log stdlib compatibility
func (p *PackageLogger) Println(args ...interface{}) {
p.internalLog(calldepth, INFO, fmt.Sprintln(args...))
}
func (p *PackageLogger) Printf(format string, args ...interface{}) {
p.internalLog(calldepth, INFO, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Print(args ...interface{}) {
p.internalLog(calldepth, INFO, fmt.Sprint(args...))
}
// Panic and fatal
func (p *PackageLogger) Panicf(format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
p.internalLog(calldepth, CRITICAL, s)
panic(s)
}
func (p *PackageLogger) Panic(args ...interface{}) {
s := fmt.Sprint(args...)
p.internalLog(calldepth, CRITICAL, s)
panic(s)
}
func (p *PackageLogger) Fatalf(format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
p.internalLog(calldepth, CRITICAL, s)
os.Exit(1)
}
func (p *PackageLogger) Fatal(args ...interface{}) {
s := fmt.Sprint(args...)
p.internalLog(calldepth, CRITICAL, s)
os.Exit(1)
}
// Error Functions
func (p *PackageLogger) Errorf(format string, args ...interface{}) {
p.internalLog(calldepth, ERROR, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Error(entries ...interface{}) {
p.internalLog(calldepth, ERROR, entries...)
}
// Warning Functions
func (p *PackageLogger) Warningf(format string, args ...interface{}) {
p.internalLog(calldepth, WARNING, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Warning(entries ...interface{}) {
p.internalLog(calldepth, WARNING, entries...)
}
// Notice Functions
func (p *PackageLogger) Noticef(format string, args ...interface{}) {
p.internalLog(calldepth, NOTICE, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Notice(entries ...interface{}) {
p.internalLog(calldepth, NOTICE, entries...)
}
// Info Functions
func (p *PackageLogger) Infof(format string, args ...interface{}) {
p.internalLog(calldepth, INFO, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Info(entries ...interface{}) {
p.internalLog(calldepth, INFO, entries...)
}
// Debug Functions
func (p *PackageLogger) Debugf(format string, args ...interface{}) {
p.internalLog(calldepth, DEBUG, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Debug(entries ...interface{}) {
p.internalLog(calldepth, DEBUG, entries...)
}
// Trace Functions
func (p *PackageLogger) Tracef(format string, args ...interface{}) {
p.internalLog(calldepth, TRACE, fmt.Sprintf(format, args...))
}
func (p *PackageLogger) Trace(entries ...interface{}) {
p.internalLog(calldepth, TRACE, entries...)
}
func (p *PackageLogger) Flush() {
logger.Lock()
defer logger.Unlock()
logger.formatter.Flush()
}

@ -0,0 +1,65 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// +build !windows
package capnslog
import (
"fmt"
"log/syslog"
)
func NewSyslogFormatter(w *syslog.Writer) Formatter {
return &syslogFormatter{w}
}
func NewDefaultSyslogFormatter(tag string) (Formatter, error) {
w, err := syslog.New(syslog.LOG_DEBUG, tag)
if err != nil {
return nil, err
}
return NewSyslogFormatter(w), nil
}
type syslogFormatter struct {
w *syslog.Writer
}
func (s *syslogFormatter) Format(pkg string, l LogLevel, _ int, entries ...interface{}) {
for _, entry := range entries {
str := fmt.Sprint(entry)
switch l {
case CRITICAL:
s.w.Crit(str)
case ERROR:
s.w.Err(str)
case WARNING:
s.w.Warning(str)
case NOTICE:
s.w.Notice(str)
case INFO:
s.w.Info(str)
case DEBUG:
s.w.Debug(str)
case TRACE:
s.w.Debug(str)
default:
panic("Unhandled loglevel")
}
}
}
func (s *syslogFormatter) Flush() {
}

@ -0,0 +1,5 @@
language: go
go:
- 1.6
- tip

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2013 Fatih Arslan
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,154 @@
# Color [![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/fatih/color) [![Build Status](http://img.shields.io/travis/fatih/color.svg?style=flat-square)](https://travis-ci.org/fatih/color)
Color lets you use colorized outputs in terms of [ANSI Escape
Codes](http://en.wikipedia.org/wiki/ANSI_escape_code#Colors) in Go (Golang). It
has support for Windows too! The API can be used in several ways, pick one that
suits you.
![Color](http://i.imgur.com/c1JI0lA.png)
## Install
```bash
go get github.com/fatih/color
```
## Examples
### Standard colors
```go
// Print with default helper functions
color.Cyan("Prints text in cyan.")
// A newline will be appended automatically
color.Blue("Prints %s in blue.", "text")
// These are using the default foreground colors
color.Red("We have red")
color.Magenta("And many others ..")
```
### Mix and reuse colors
```go
// Create a new color object
c := color.New(color.FgCyan).Add(color.Underline)
c.Println("Prints cyan text with an underline.")
// Or just add them to New()
d := color.New(color.FgCyan, color.Bold)
d.Printf("This prints bold cyan %s\n", "too!.")
// Mix up foreground and background colors, create new mixes!
red := color.New(color.FgRed)
boldRed := red.Add(color.Bold)
boldRed.Println("This will print text in bold red.")
whiteBackground := red.Add(color.BgWhite)
whiteBackground.Println("Red text with white background.")
```
### Custom print functions (PrintFunc)
```go
// Create a custom print function for convenience
red := color.New(color.FgRed).PrintfFunc()
red("Warning")
red("Error: %s", err)
// Mix up multiple attributes
notice := color.New(color.Bold, color.FgGreen).PrintlnFunc()
notice("Don't forget this...")
```
### Insert into noncolor strings (SprintFunc)
```go
// Create SprintXxx functions to mix strings with other non-colorized strings:
yellow := color.New(color.FgYellow).SprintFunc()
red := color.New(color.FgRed).SprintFunc()
fmt.Printf("This is a %s and this is %s.\n", yellow("warning"), red("error"))
info := color.New(color.FgWhite, color.BgGreen).SprintFunc()
fmt.Printf("This %s rocks!\n", info("package"))
// Use helper functions
fmt.Printf("This", color.RedString("warning"), "should be not neglected.")
fmt.Printf(color.GreenString("Info:"), "an important message." )
// Windows supported too! Just don't forget to change the output to color.Output
fmt.Fprintf(color.Output, "Windows support: %s", color.GreenString("PASS"))
```
### Plug into existing code
```go
// Use handy standard colors
color.Set(color.FgYellow)
fmt.Println("Existing text will now be in yellow")
fmt.Printf("This one %s\n", "too")
color.Unset() // Don't forget to unset
// You can mix up parameters
color.Set(color.FgMagenta, color.Bold)
defer color.Unset() // Use it in your function
fmt.Println("All text will now be bold magenta.")
```
### Disable color
There might be a case where you want to disable color output (for example to
pipe the standard output of your app to somewhere else). `Color` has support to
disable colors both globally and for single color definition. For example
suppose you have a CLI app and a `--no-color` bool flag. You can easily disable
the color output with:
```go
var flagNoColor = flag.Bool("no-color", false, "Disable color output")
if *flagNoColor {
color.NoColor = true // disables colorized output
}
```
It also has support for single color definitions (local). You can
disable/enable color output on the fly:
```go
c := color.New(color.FgCyan)
c.Println("Prints cyan text")
c.DisableColor()
c.Println("This is printed without any color")
c.EnableColor()
c.Println("This prints again cyan...")
```
## Todo
* Save/Return previous values
* Evaluate fmt.Formatter interface
## Credits
* [Fatih Arslan](https://github.com/fatih)
* Windows support via @mattn: [colorable](https://github.com/mattn/go-colorable)
## License
The MIT License (MIT) - see [`LICENSE.md`](https://github.com/fatih/color/blob/master/LICENSE.md) for more details

@ -0,0 +1,402 @@
package color
import (
"fmt"
"os"
"strconv"
"strings"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
)
// NoColor defines if the output is colorized or not. It's dynamically set to
// false or true based on the stdout's file descriptor referring to a terminal
// or not. This is a global option and affects all colors. For more control
// over each color block use the methods DisableColor() individually.
var NoColor = !isatty.IsTerminal(os.Stdout.Fd())
// Color defines a custom color object which is defined by SGR parameters.
type Color struct {
params []Attribute
noColor *bool
}
// Attribute defines a single SGR Code
type Attribute int
const escape = "\x1b"
// Base attributes
const (
Reset Attribute = iota
Bold
Faint
Italic
Underline
BlinkSlow
BlinkRapid
ReverseVideo
Concealed
CrossedOut
)
// Foreground text colors
const (
FgBlack Attribute = iota + 30
FgRed
FgGreen
FgYellow
FgBlue
FgMagenta
FgCyan
FgWhite
)
// Foreground Hi-Intensity text colors
const (
FgHiBlack Attribute = iota + 90
FgHiRed
FgHiGreen
FgHiYellow
FgHiBlue
FgHiMagenta
FgHiCyan
FgHiWhite
)
// Background text colors
const (
BgBlack Attribute = iota + 40
BgRed
BgGreen
BgYellow
BgBlue
BgMagenta
BgCyan
BgWhite
)
// Background Hi-Intensity text colors
const (
BgHiBlack Attribute = iota + 100
BgHiRed
BgHiGreen
BgHiYellow
BgHiBlue
BgHiMagenta
BgHiCyan
BgHiWhite
)
// New returns a newly created color object.
func New(value ...Attribute) *Color {
c := &Color{params: make([]Attribute, 0)}
c.Add(value...)
return c
}
// Set sets the given parameters immediately. It will change the color of
// output with the given SGR parameters until color.Unset() is called.
func Set(p ...Attribute) *Color {
c := New(p...)
c.Set()
return c
}
// Unset resets all escape attributes and clears the output. Usually should
// be called after Set().
func Unset() {
if NoColor {
return
}
fmt.Fprintf(Output, "%s[%dm", escape, Reset)
}
// Set sets the SGR sequence.
func (c *Color) Set() *Color {
if c.isNoColorSet() {
return c
}
fmt.Fprintf(Output, c.format())
return c
}
func (c *Color) unset() {
if c.isNoColorSet() {
return
}
Unset()
}
// Add is used to chain SGR parameters. Use as many as parameters to combine
// and create custom color objects. Example: Add(color.FgRed, color.Underline).
func (c *Color) Add(value ...Attribute) *Color {
c.params = append(c.params, value...)
return c
}
func (c *Color) prepend(value Attribute) {
c.params = append(c.params, 0)
copy(c.params[1:], c.params[0:])
c.params[0] = value
}
// Output defines the standard output of the print functions. By default
// os.Stdout is used.
var Output = colorable.NewColorableStdout()
// Print formats using the default formats for its operands and writes to
// standard output. Spaces are added between operands when neither is a
// string. It returns the number of bytes written and any write error
// encountered. This is the standard fmt.Print() method wrapped with the given
// color.
func (c *Color) Print(a ...interface{}) (n int, err error) {
c.Set()
defer c.unset()
return fmt.Fprint(Output, a...)
}
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
// This is the standard fmt.Printf() method wrapped with the given color.
func (c *Color) Printf(format string, a ...interface{}) (n int, err error) {
c.Set()
defer c.unset()
return fmt.Fprintf(Output, format, a...)
}
// Println formats using the default formats for its operands and writes to
// standard output. Spaces are always added between operands and a newline is
// appended. It returns the number of bytes written and any write error
// encountered. This is the standard fmt.Print() method wrapped with the given
// color.
func (c *Color) Println(a ...interface{}) (n int, err error) {
c.Set()
defer c.unset()
return fmt.Fprintln(Output, a...)
}
// PrintFunc returns a new function that prints the passed arguments as
// colorized with color.Print().
func (c *Color) PrintFunc() func(a ...interface{}) {
return func(a ...interface{}) { c.Print(a...) }
}
// PrintfFunc returns a new function that prints the passed arguments as
// colorized with color.Printf().
func (c *Color) PrintfFunc() func(format string, a ...interface{}) {
return func(format string, a ...interface{}) { c.Printf(format, a...) }
}
// PrintlnFunc returns a new function that prints the passed arguments as
// colorized with color.Println().
func (c *Color) PrintlnFunc() func(a ...interface{}) {
return func(a ...interface{}) { c.Println(a...) }
}
// SprintFunc returns a new function that returns colorized strings for the
// given arguments with fmt.Sprint(). Useful to put into or mix into other
// string. Windows users should use this in conjuction with color.Output, example:
//
// put := New(FgYellow).SprintFunc()
// fmt.Fprintf(color.Output, "This is a %s", put("warning"))
func (c *Color) SprintFunc() func(a ...interface{}) string {
return func(a ...interface{}) string {
return c.wrap(fmt.Sprint(a...))
}
}
// SprintfFunc returns a new function that returns colorized strings for the
// given arguments with fmt.Sprintf(). Useful to put into or mix into other
// string. Windows users should use this in conjuction with color.Output.
func (c *Color) SprintfFunc() func(format string, a ...interface{}) string {
return func(format string, a ...interface{}) string {
return c.wrap(fmt.Sprintf(format, a...))
}
}
// SprintlnFunc returns a new function that returns colorized strings for the
// given arguments with fmt.Sprintln(). Useful to put into or mix into other
// string. Windows users should use this in conjuction with color.Output.
func (c *Color) SprintlnFunc() func(a ...interface{}) string {
return func(a ...interface{}) string {
return c.wrap(fmt.Sprintln(a...))
}
}
// sequence returns a formated SGR sequence to be plugged into a "\x1b[...m"
// an example output might be: "1;36" -> bold cyan
func (c *Color) sequence() string {
format := make([]string, len(c.params))
for i, v := range c.params {
format[i] = strconv.Itoa(int(v))
}
return strings.Join(format, ";")
}
// wrap wraps the s string with the colors attributes. The string is ready to
// be printed.
func (c *Color) wrap(s string) string {
if c.isNoColorSet() {
return s
}
return c.format() + s + c.unformat()
}
func (c *Color) format() string {
return fmt.Sprintf("%s[%sm", escape, c.sequence())
}
func (c *Color) unformat() string {
return fmt.Sprintf("%s[%dm", escape, Reset)
}
// DisableColor disables the color output. Useful to not change any existing
// code and still being able to output. Can be used for flags like
// "--no-color". To enable back use EnableColor() method.
func (c *Color) DisableColor() {
c.noColor = boolPtr(true)
}
// EnableColor enables the color output. Use it in conjuction with
// DisableColor(). Otherwise this method has no side effects.
func (c *Color) EnableColor() {
c.noColor = boolPtr(false)
}
func (c *Color) isNoColorSet() bool {
// check first if we have user setted action
if c.noColor != nil {
return *c.noColor
}
// if not return the global option, which is disabled by default
return NoColor
}
// Equals returns a boolean value indicating whether two colors are equal.
func (c *Color) Equals(c2 *Color) bool {
if len(c.params) != len(c2.params) {
return false
}
for _, attr := range c.params {
if !c2.attrExists(attr) {
return false
}
}
return true
}
func (c *Color) attrExists(a Attribute) bool {
for _, attr := range c.params {
if attr == a {
return true
}
}
return false
}
func boolPtr(v bool) *bool {
return &v
}
// Black is an convenient helper function to print with black foreground. A
// newline is appended to format by default.
func Black(format string, a ...interface{}) { printColor(format, FgBlack, a...) }
// Red is an convenient helper function to print with red foreground. A
// newline is appended to format by default.
func Red(format string, a ...interface{}) { printColor(format, FgRed, a...) }
// Green is an convenient helper function to print with green foreground. A
// newline is appended to format by default.
func Green(format string, a ...interface{}) { printColor(format, FgGreen, a...) }
// Yellow is an convenient helper function to print with yellow foreground.
// A newline is appended to format by default.
func Yellow(format string, a ...interface{}) { printColor(format, FgYellow, a...) }
// Blue is an convenient helper function to print with blue foreground. A
// newline is appended to format by default.
func Blue(format string, a ...interface{}) { printColor(format, FgBlue, a...) }
// Magenta is an convenient helper function to print with magenta foreground.
// A newline is appended to format by default.
func Magenta(format string, a ...interface{}) { printColor(format, FgMagenta, a...) }
// Cyan is an convenient helper function to print with cyan foreground. A
// newline is appended to format by default.
func Cyan(format string, a ...interface{}) { printColor(format, FgCyan, a...) }
// White is an convenient helper function to print with white foreground. A
// newline is appended to format by default.
func White(format string, a ...interface{}) { printColor(format, FgWhite, a...) }
func printColor(format string, p Attribute, a ...interface{}) {
if !strings.HasSuffix(format, "\n") {
format += "\n"
}
c := &Color{params: []Attribute{p}}
c.Printf(format, a...)
}
// BlackString is an convenient helper function to return a string with black
// foreground.
func BlackString(format string, a ...interface{}) string {
return New(FgBlack).SprintfFunc()(format, a...)
}
// RedString is an convenient helper function to return a string with red
// foreground.
func RedString(format string, a ...interface{}) string {
return New(FgRed).SprintfFunc()(format, a...)
}
// GreenString is an convenient helper function to return a string with green
// foreground.
func GreenString(format string, a ...interface{}) string {
return New(FgGreen).SprintfFunc()(format, a...)
}
// YellowString is an convenient helper function to return a string with yellow
// foreground.
func YellowString(format string, a ...interface{}) string {
return New(FgYellow).SprintfFunc()(format, a...)
}
// BlueString is an convenient helper function to return a string with blue
// foreground.
func BlueString(format string, a ...interface{}) string {
return New(FgBlue).SprintfFunc()(format, a...)
}
// MagentaString is an convenient helper function to return a string with magenta
// foreground.
func MagentaString(format string, a ...interface{}) string {
return New(FgMagenta).SprintfFunc()(format, a...)
}
// CyanString is an convenient helper function to return a string with cyan
// foreground.
func CyanString(format string, a ...interface{}) string {
return New(FgCyan).SprintfFunc()(format, a...)
}
// WhiteString is an convenient helper function to return a string with white
// foreground.
func WhiteString(format string, a ...interface{}) string {
return New(FgWhite).SprintfFunc()(format, a...)
}

@ -0,0 +1,114 @@
/*
Package color is an ANSI color package to output colorized or SGR defined
output to the standard output. The API can be used in several way, pick one
that suits you.
Use simple and default helper functions with predefined foreground colors:
color.Cyan("Prints text in cyan.")
// a newline will be appended automatically
color.Blue("Prints %s in blue.", "text")
// More default foreground colors..
color.Red("We have red")
color.Yellow("Yellow color too!")
color.Magenta("And many others ..")
However there are times where custom color mixes are required. Below are some
examples to create custom color objects and use the print functions of each
separate color object.
// Create a new color object
c := color.New(color.FgCyan).Add(color.Underline)
c.Println("Prints cyan text with an underline.")
// Or just add them to New()
d := color.New(color.FgCyan, color.Bold)
d.Printf("This prints bold cyan %s\n", "too!.")
// Mix up foreground and background colors, create new mixes!
red := color.New(color.FgRed)
boldRed := red.Add(color.Bold)
boldRed.Println("This will print text in bold red.")
whiteBackground := red.Add(color.BgWhite)
whiteBackground.Println("Red text with White background.")
You can create PrintXxx functions to simplify even more:
// Create a custom print function for convenient
red := color.New(color.FgRed).PrintfFunc()
red("warning")
red("error: %s", err)
// Mix up multiple attributes
notice := color.New(color.Bold, color.FgGreen).PrintlnFunc()
notice("don't forget this...")
Or create SprintXxx functions to mix strings with other non-colorized strings:
yellow := New(FgYellow).SprintFunc()
red := New(FgRed).SprintFunc()
fmt.Printf("this is a %s and this is %s.\n", yellow("warning"), red("error"))
info := New(FgWhite, BgGreen).SprintFunc()
fmt.Printf("this %s rocks!\n", info("package"))
Windows support is enabled by default. All Print functions works as intended.
However only for color.SprintXXX functions, user should use fmt.FprintXXX and
set the output to color.Output:
fmt.Fprintf(color.Output, "Windows support: %s", color.GreenString("PASS"))
info := New(FgWhite, BgGreen).SprintFunc()
fmt.Fprintf(color.Output, "this %s rocks!\n", info("package"))
Using with existing code is possible. Just use the Set() method to set the
standard output to the given parameters. That way a rewrite of an existing
code is not required.
// Use handy standard colors.
color.Set(color.FgYellow)
fmt.Println("Existing text will be now in Yellow")
fmt.Printf("This one %s\n", "too")
color.Unset() // don't forget to unset
// You can mix up parameters
color.Set(color.FgMagenta, color.Bold)
defer color.Unset() // use it in your function
fmt.Println("All text will be now bold magenta.")
There might be a case where you want to disable color output (for example to
pipe the standard output of your app to somewhere else). `Color` has support to
disable colors both globally and for single color definition. For example
suppose you have a CLI app and a `--no-color` bool flag. You can easily disable
the color output with:
var flagNoColor = flag.Bool("no-color", false, "Disable color output")
if *flagNoColor {
color.NoColor = true // disables colorized output
}
It also has support for single color definitions (local). You can
disable/enable color output on the fly:
c := color.New(color.FgCyan)
c.Println("Prints cyan text")
c.DisableColor()
c.Println("This is printed without any color")
c.EnableColor()
c.Println("This prints again cyan...")
*/
package color

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Yasuhiro Matsumoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

@ -0,0 +1,43 @@
# go-colorable
Colorable writer for windows.
For example, most of logger packages doesn't show colors on windows. (I know we can do it with ansicon. But I don't want.)
This package is possible to handle escape sequence for ansi color on windows.
## Too Bad!
![](https://raw.githubusercontent.com/mattn/go-colorable/gh-pages/bad.png)
## So Good!
![](https://raw.githubusercontent.com/mattn/go-colorable/gh-pages/good.png)
## Usage
```go
logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true})
logrus.SetOutput(colorable.NewColorableStdout())
logrus.Info("succeeded")
logrus.Warn("not correct")
logrus.Error("something error")
logrus.Fatal("panic")
```
You can compile above code on non-windows OSs.
## Installation
```
$ go get github.com/mattn/go-colorable
```
# License
MIT
# Author
Yasuhiro Matsumoto (a.k.a mattn)

@ -0,0 +1,24 @@
// +build !windows
package colorable
import (
"io"
"os"
)
func NewColorable(file *os.File) io.Writer {
if file == nil {
panic("nil passed instead of *os.File to NewColorable()")
}
return file
}
func NewColorableStdout() io.Writer {
return os.Stdout
}
func NewColorableStderr() io.Writer {
return os.Stderr
}

@ -0,0 +1,783 @@
package colorable
import (
"bytes"
"fmt"
"io"
"math"
"os"
"strconv"
"strings"
"syscall"
"unsafe"
"github.com/mattn/go-isatty"
)
const (
foregroundBlue = 0x1
foregroundGreen = 0x2
foregroundRed = 0x4
foregroundIntensity = 0x8
foregroundMask = (foregroundRed | foregroundBlue | foregroundGreen | foregroundIntensity)
backgroundBlue = 0x10
backgroundGreen = 0x20
backgroundRed = 0x40
backgroundIntensity = 0x80
backgroundMask = (backgroundRed | backgroundBlue | backgroundGreen | backgroundIntensity)
)
type wchar uint16
type short int16
type dword uint32
type word uint16
type coord struct {
x short
y short
}
type smallRect struct {
left short
top short
right short
bottom short
}
type consoleScreenBufferInfo struct {
size coord
cursorPosition coord
attributes word
window smallRect
maximumWindowSize coord
}
var (
kernel32 = syscall.NewLazyDLL("kernel32.dll")
procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
procSetConsoleTextAttribute = kernel32.NewProc("SetConsoleTextAttribute")
procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition")
procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW")
procFillConsoleOutputAttribute = kernel32.NewProc("FillConsoleOutputAttribute")
)
type Writer struct {
out io.Writer
handle syscall.Handle
lastbuf bytes.Buffer
oldattr word
}
func NewColorable(file *os.File) io.Writer {
if file == nil {
panic("nil passed instead of *os.File to NewColorable()")
}
if isatty.IsTerminal(file.Fd()) {
var csbi consoleScreenBufferInfo
handle := syscall.Handle(file.Fd())
procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi)))
return &Writer{out: file, handle: handle, oldattr: csbi.attributes}
} else {
return file
}
}
func NewColorableStdout() io.Writer {
return NewColorable(os.Stdout)
}
func NewColorableStderr() io.Writer {
return NewColorable(os.Stderr)
}
var color256 = map[int]int{
0: 0x000000,
1: 0x800000,
2: 0x008000,
3: 0x808000,
4: 0x000080,
5: 0x800080,
6: 0x008080,
7: 0xc0c0c0,
8: 0x808080,
9: 0xff0000,
10: 0x00ff00,
11: 0xffff00,
12: 0x0000ff,
13: 0xff00ff,
14: 0x00ffff,
15: 0xffffff,
16: 0x000000,
17: 0x00005f,
18: 0x000087,
19: 0x0000af,
20: 0x0000d7,
21: 0x0000ff,
22: 0x005f00,
23: 0x005f5f,
24: 0x005f87,
25: 0x005faf,
26: 0x005fd7,
27: 0x005fff,
28: 0x008700,
29: 0x00875f,
30: 0x008787,
31: 0x0087af,
32: 0x0087d7,
33: 0x0087ff,
34: 0x00af00,
35: 0x00af5f,
36: 0x00af87,
37: 0x00afaf,
38: 0x00afd7,
39: 0x00afff,
40: 0x00d700,
41: 0x00d75f,
42: 0x00d787,
43: 0x00d7af,
44: 0x00d7d7,
45: 0x00d7ff,
46: 0x00ff00,
47: 0x00ff5f,
48: 0x00ff87,
49: 0x00ffaf,
50: 0x00ffd7,
51: 0x00ffff,
52: 0x5f0000,
53: 0x5f005f,
54: 0x5f0087,
55: 0x5f00af,
56: 0x5f00d7,
57: 0x5f00ff,
58: 0x5f5f00,
59: 0x5f5f5f,
60: 0x5f5f87,
61: 0x5f5faf,
62: 0x5f5fd7,
63: 0x5f5fff,
64: 0x5f8700,
65: 0x5f875f,
66: 0x5f8787,
67: 0x5f87af,
68: 0x5f87d7,
69: 0x5f87ff,
70: 0x5faf00,
71: 0x5faf5f,
72: 0x5faf87,
73: 0x5fafaf,
74: 0x5fafd7,
75: 0x5fafff,
76: 0x5fd700,
77: 0x5fd75f,
78: 0x5fd787,
79: 0x5fd7af,
80: 0x5fd7d7,
81: 0x5fd7ff,
82: 0x5fff00,
83: 0x5fff5f,
84: 0x5fff87,
85: 0x5fffaf,
86: 0x5fffd7,
87: 0x5fffff,
88: 0x870000,
89: 0x87005f,
90: 0x870087,
91: 0x8700af,
92: 0x8700d7,
93: 0x8700ff,
94: 0x875f00,
95: 0x875f5f,
96: 0x875f87,
97: 0x875faf,
98: 0x875fd7,
99: 0x875fff,
100: 0x878700,
101: 0x87875f,
102: 0x878787,
103: 0x8787af,
104: 0x8787d7,
105: 0x8787ff,
106: 0x87af00,
107: 0x87af5f,
108: 0x87af87,
109: 0x87afaf,
110: 0x87afd7,
111: 0x87afff,
112: 0x87d700,
113: 0x87d75f,
114: 0x87d787,
115: 0x87d7af,
116: 0x87d7d7,
117: 0x87d7ff,
118: 0x87ff00,
119: 0x87ff5f,
120: 0x87ff87,
121: 0x87ffaf,
122: 0x87ffd7,
123: 0x87ffff,
124: 0xaf0000,
125: 0xaf005f,
126: 0xaf0087,
127: 0xaf00af,
128: 0xaf00d7,
129: 0xaf00ff,
130: 0xaf5f00,
131: 0xaf5f5f,
132: 0xaf5f87,
133: 0xaf5faf,
134: 0xaf5fd7,
135: 0xaf5fff,
136: 0xaf8700,
137: 0xaf875f,
138: 0xaf8787,
139: 0xaf87af,
140: 0xaf87d7,
141: 0xaf87ff,
142: 0xafaf00,
143: 0xafaf5f,
144: 0xafaf87,
145: 0xafafaf,
146: 0xafafd7,
147: 0xafafff,
148: 0xafd700,
149: 0xafd75f,
150: 0xafd787,
151: 0xafd7af,
152: 0xafd7d7,
153: 0xafd7ff,
154: 0xafff00,
155: 0xafff5f,
156: 0xafff87,
157: 0xafffaf,
158: 0xafffd7,
159: 0xafffff,
160: 0xd70000,
161: 0xd7005f,
162: 0xd70087,
163: 0xd700af,
164: 0xd700d7,
165: 0xd700ff,
166: 0xd75f00,
167: 0xd75f5f,
168: 0xd75f87,
169: 0xd75faf,
170: 0xd75fd7,
171: 0xd75fff,
172: 0xd78700,
173: 0xd7875f,
174: 0xd78787,
175: 0xd787af,
176: 0xd787d7,
177: 0xd787ff,
178: 0xd7af00,
179: 0xd7af5f,
180: 0xd7af87,
181: 0xd7afaf,
182: 0xd7afd7,
183: 0xd7afff,
184: 0xd7d700,
185: 0xd7d75f,
186: 0xd7d787,
187: 0xd7d7af,
188: 0xd7d7d7,
189: 0xd7d7ff,
190: 0xd7ff00,
191: 0xd7ff5f,
192: 0xd7ff87,
193: 0xd7ffaf,
194: 0xd7ffd7,
195: 0xd7ffff,
196: 0xff0000,
197: 0xff005f,
198: 0xff0087,
199: 0xff00af,
200: 0xff00d7,
201: 0xff00ff,
202: 0xff5f00,
203: 0xff5f5f,
204: 0xff5f87,
205: 0xff5faf,
206: 0xff5fd7,
207: 0xff5fff,
208: 0xff8700,
209: 0xff875f,
210: 0xff8787,
211: 0xff87af,
212: 0xff87d7,
213: 0xff87ff,
214: 0xffaf00,
215: 0xffaf5f,
216: 0xffaf87,
217: 0xffafaf,
218: 0xffafd7,
219: 0xffafff,
220: 0xffd700,
221: 0xffd75f,
222: 0xffd787,
223: 0xffd7af,
224: 0xffd7d7,
225: 0xffd7ff,
226: 0xffff00,
227: 0xffff5f,
228: 0xffff87,
229: 0xffffaf,
230: 0xffffd7,
231: 0xffffff,
232: 0x080808,
233: 0x121212,
234: 0x1c1c1c,
235: 0x262626,
236: 0x303030,
237: 0x3a3a3a,
238: 0x444444,
239: 0x4e4e4e,
240: 0x585858,
241: 0x626262,
242: 0x6c6c6c,
243: 0x767676,
244: 0x808080,
245: 0x8a8a8a,
246: 0x949494,
247: 0x9e9e9e,
248: 0xa8a8a8,
249: 0xb2b2b2,
250: 0xbcbcbc,
251: 0xc6c6c6,
252: 0xd0d0d0,
253: 0xdadada,
254: 0xe4e4e4,
255: 0xeeeeee,
}
func (w *Writer) Write(data []byte) (n int, err error) {
var csbi consoleScreenBufferInfo
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
er := bytes.NewBuffer(data)
loop:
for {
r1, _, err := procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
if r1 == 0 {
break loop
}
c1, _, err := er.ReadRune()
if err != nil {
break loop
}
if c1 != 0x1b {
fmt.Fprint(w.out, string(c1))
continue
}
c2, _, err := er.ReadRune()
if err != nil {
w.lastbuf.WriteRune(c1)
break loop
}
if c2 != 0x5b {
w.lastbuf.WriteRune(c1)
w.lastbuf.WriteRune(c2)
continue
}
var buf bytes.Buffer
var m rune
for {
c, _, err := er.ReadRune()
if err != nil {
w.lastbuf.WriteRune(c1)
w.lastbuf.WriteRune(c2)
w.lastbuf.Write(buf.Bytes())
break loop
}
if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '@' {
m = c
break
}
buf.Write([]byte(string(c)))
}
var csbi consoleScreenBufferInfo
switch m {
case 'A':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.y -= short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'B':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.y += short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'C':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x -= short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'D':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
if n, err = strconv.Atoi(buf.String()); err == nil {
var csbi consoleScreenBufferInfo
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x += short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
}
case 'E':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x = 0
csbi.cursorPosition.y += short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'F':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x = 0
csbi.cursorPosition.y -= short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'G':
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x = short(n)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'H':
token := strings.Split(buf.String(), ";")
if len(token) != 2 {
continue
}
n1, err := strconv.Atoi(token[0])
if err != nil {
continue
}
n2, err := strconv.Atoi(token[1])
if err != nil {
continue
}
csbi.cursorPosition.x = short(n2)
csbi.cursorPosition.x = short(n1)
procSetConsoleCursorPosition.Call(uintptr(w.handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'J':
n, err := strconv.Atoi(buf.String())
if err != nil {
continue
}
var cursor coord
switch n {
case 0:
cursor = coord{x: csbi.cursorPosition.x, y: csbi.cursorPosition.y}
case 1:
cursor = coord{x: csbi.window.left, y: csbi.window.top}
case 2:
cursor = coord{x: csbi.window.left, y: csbi.window.top}
}
var count, written dword
count = dword(csbi.size.x - csbi.cursorPosition.x + (csbi.size.y-csbi.cursorPosition.y)*csbi.size.x)
procFillConsoleOutputCharacter.Call(uintptr(w.handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
procFillConsoleOutputAttribute.Call(uintptr(w.handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
case 'K':
n, err := strconv.Atoi(buf.String())
if err != nil {
continue
}
var cursor coord
switch n {
case 0:
cursor = coord{x: csbi.cursorPosition.x, y: csbi.cursorPosition.y}
case 1:
cursor = coord{x: csbi.window.left, y: csbi.window.top + csbi.cursorPosition.y}
case 2:
cursor = coord{x: csbi.window.left, y: csbi.window.top + csbi.cursorPosition.y}
}
var count, written dword
count = dword(csbi.size.x - csbi.cursorPosition.x)
procFillConsoleOutputCharacter.Call(uintptr(w.handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
procFillConsoleOutputAttribute.Call(uintptr(w.handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
case 'm':
attr := csbi.attributes
cs := buf.String()
if cs == "" {
procSetConsoleTextAttribute.Call(uintptr(w.handle), uintptr(w.oldattr))
continue
}
token := strings.Split(cs, ";")
for i := 0; i < len(token); i += 1 {
ns := token[i]
if n, err = strconv.Atoi(ns); err == nil {
switch {
case n == 0 || n == 100:
attr = w.oldattr
case 1 <= n && n <= 5:
attr |= foregroundIntensity
case n == 7:
attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4)
case 22 == n || n == 25 || n == 25:
attr |= foregroundIntensity
case n == 27:
attr = ((attr & foregroundMask) << 4) | ((attr & backgroundMask) >> 4)
case 30 <= n && n <= 37:
attr = (attr & backgroundMask)
if (n-30)&1 != 0 {
attr |= foregroundRed
}
if (n-30)&2 != 0 {
attr |= foregroundGreen
}
if (n-30)&4 != 0 {
attr |= foregroundBlue
}
case n == 38: // set foreground color.
if i < len(token)-2 && (token[i+1] == "5" || token[i+1] == "05") {
if n256, err := strconv.Atoi(token[i+2]); err == nil {
if n256foreAttr == nil {
n256setup()
}
attr &= backgroundMask
attr |= n256foreAttr[n256]
i += 2
}
} else {
attr = attr & (w.oldattr & backgroundMask)
}
case n == 39: // reset foreground color.
attr &= backgroundMask
attr |= w.oldattr & foregroundMask
case 40 <= n && n <= 47:
attr = (attr & foregroundMask)
if (n-40)&1 != 0 {
attr |= backgroundRed
}
if (n-40)&2 != 0 {
attr |= backgroundGreen
}
if (n-40)&4 != 0 {
attr |= backgroundBlue
}
case n == 48: // set background color.
if i < len(token)-2 && token[i+1] == "5" {
if n256, err := strconv.Atoi(token[i+2]); err == nil {
if n256backAttr == nil {
n256setup()
}
attr &= foregroundMask
attr |= n256backAttr[n256]
i += 2
}
} else {
attr = attr & (w.oldattr & foregroundMask)
}
case n == 49: // reset foreground color.
attr &= foregroundMask
attr |= w.oldattr & backgroundMask
case 90 <= n && n <= 97:
attr = (attr & backgroundMask)
attr |= foregroundIntensity
if (n-90)&1 != 0 {
attr |= foregroundRed
}
if (n-90)&2 != 0 {
attr |= foregroundGreen
}
if (n-90)&4 != 0 {
attr |= foregroundBlue
}
case 100 <= n && n <= 107:
attr = (attr & foregroundMask)
attr |= backgroundIntensity
if (n-100)&1 != 0 {
attr |= backgroundRed
}
if (n-100)&2 != 0 {
attr |= backgroundGreen
}
if (n-100)&4 != 0 {
attr |= backgroundBlue
}
}
procSetConsoleTextAttribute.Call(uintptr(w.handle), uintptr(attr))
}
}
}
}
return len(data) - w.lastbuf.Len(), nil
}
type consoleColor struct {
rgb int
red bool
green bool
blue bool
intensity bool
}
func (c consoleColor) foregroundAttr() (attr word) {
if c.red {
attr |= foregroundRed
}
if c.green {
attr |= foregroundGreen
}
if c.blue {
attr |= foregroundBlue
}
if c.intensity {
attr |= foregroundIntensity
}
return
}
func (c consoleColor) backgroundAttr() (attr word) {
if c.red {
attr |= backgroundRed
}
if c.green {
attr |= backgroundGreen
}
if c.blue {
attr |= backgroundBlue
}
if c.intensity {
attr |= backgroundIntensity
}
return
}
var color16 = []consoleColor{
consoleColor{0x000000, false, false, false, false},
consoleColor{0x000080, false, false, true, false},
consoleColor{0x008000, false, true, false, false},
consoleColor{0x008080, false, true, true, false},
consoleColor{0x800000, true, false, false, false},
consoleColor{0x800080, true, false, true, false},
consoleColor{0x808000, true, true, false, false},
consoleColor{0xc0c0c0, true, true, true, false},
consoleColor{0x808080, false, false, false, true},
consoleColor{0x0000ff, false, false, true, true},
consoleColor{0x00ff00, false, true, false, true},
consoleColor{0x00ffff, false, true, true, true},
consoleColor{0xff0000, true, false, false, true},
consoleColor{0xff00ff, true, false, true, true},
consoleColor{0xffff00, true, true, false, true},
consoleColor{0xffffff, true, true, true, true},
}
type hsv struct {
h, s, v float32
}
func (a hsv) dist(b hsv) float32 {
dh := a.h - b.h
switch {
case dh > 0.5:
dh = 1 - dh
case dh < -0.5:
dh = -1 - dh
}
ds := a.s - b.s
dv := a.v - b.v
return float32(math.Sqrt(float64(dh*dh + ds*ds + dv*dv)))
}
func toHSV(rgb int) hsv {
r, g, b := float32((rgb&0xFF0000)>>16)/256.0,
float32((rgb&0x00FF00)>>8)/256.0,
float32(rgb&0x0000FF)/256.0
min, max := minmax3f(r, g, b)
h := max - min
if h > 0 {
if max == r {
h = (g - b) / h
if h < 0 {
h += 6
}
} else if max == g {
h = 2 + (b-r)/h
} else {
h = 4 + (r-g)/h
}
}
h /= 6.0
s := max - min
if max != 0 {
s /= max
}
v := max
return hsv{h: h, s: s, v: v}
}
type hsvTable []hsv
func toHSVTable(rgbTable []consoleColor) hsvTable {
t := make(hsvTable, len(rgbTable))
for i, c := range rgbTable {
t[i] = toHSV(c.rgb)
}
return t
}
func (t hsvTable) find(rgb int) consoleColor {
hsv := toHSV(rgb)
n := 7
l := float32(5.0)
for i, p := range t {
d := hsv.dist(p)
if d < l {
l, n = d, i
}
}
return color16[n]
}
func minmax3f(a, b, c float32) (min, max float32) {
if a < b {
if b < c {
return a, c
} else if a < c {
return a, b
} else {
return c, b
}
} else {
if a < c {
return b, c
} else if b < c {
return b, a
} else {
return c, a
}
}
}
var n256foreAttr []word
var n256backAttr []word
func n256setup() {
n256foreAttr = make([]word, 256)
n256backAttr = make([]word, 256)
t := toHSVTable(color16)
for i, rgb := range color256 {
c := t.find(rgb)
n256foreAttr[i] = c.foregroundAttr()
n256backAttr[i] = c.backgroundAttr()
}
}

@ -0,0 +1,9 @@
Copyright (c) Yasuhiro MATSUMOTO <mattn.jp@gmail.com>
MIT License (Expat)
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,37 @@
# go-isatty
isatty for golang
## Usage
```go
package main
import (
"fmt"
"github.com/mattn/go-isatty"
"os"
)
func main() {
if isatty.IsTerminal(os.Stdout.Fd()) {
fmt.Println("Is Terminal")
} else {
fmt.Println("Is Not Terminal")
}
}
```
## Installation
```
$ go get github.com/mattn/go-isatty
```
# License
MIT
# Author
Yasuhiro Matsumoto (a.k.a mattn)

@ -0,0 +1,2 @@
// Package isatty implements interface to isatty
package isatty

@ -0,0 +1,9 @@
// +build appengine
package isatty
// IsTerminal returns true if the file descriptor is terminal which
// is always false on on appengine classic which is a sandboxed PaaS.
func IsTerminal(fd uintptr) bool {
return false
}

@ -0,0 +1,18 @@
// +build darwin freebsd openbsd netbsd
// +build !appengine
package isatty
import (
"syscall"
"unsafe"
)
const ioctlReadTermios = syscall.TIOCGETA
// IsTerminal return true if the file descriptor is terminal.
func IsTerminal(fd uintptr) bool {
var termios syscall.Termios
_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
return err == 0
}

@ -0,0 +1,18 @@
// +build linux
// +build !appengine
package isatty
import (
"syscall"
"unsafe"
)
const ioctlReadTermios = syscall.TCGETS
// IsTerminal return true if the file descriptor is terminal.
func IsTerminal(fd uintptr) bool {
var termios syscall.Termios
_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
return err == 0
}

@ -0,0 +1,16 @@
// +build solaris
// +build !appengine
package isatty
import (
"golang.org/x/sys/unix"
)
// IsTerminal returns true if the given file descriptor is a terminal.
// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c
func IsTerminal(fd uintptr) bool {
var termio unix.Termio
err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio)
return err == nil
}

@ -0,0 +1,19 @@
// +build windows
// +build !appengine
package isatty
import (
"syscall"
"unsafe"
)
var kernel32 = syscall.NewLazyDLL("kernel32.dll")
var procGetConsoleMode = kernel32.NewProc("GetConsoleMode")
// IsTerminal return true if the file descriptor is terminal.
func IsTerminal(fd uintptr) bool {
var st uint32
r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0)
return r != 0 && e == 0
}

@ -0,0 +1,17 @@
{
"version": 0,
"dependencies": [
{
"importpath": "github.com/mattn/go-colorable",
"repository": "https://github.com/mattn/go-colorable",
"revision": "9cbef7c35391cca05f15f8181dc0b18bc9736dbb",
"branch": "master"
},
{
"importpath": "github.com/mattn/go-isatty",
"repository": "https://github.com/mattn/go-isatty",
"revision": "56b76bdf51f7708750eac80fa38b952bb9f32639",
"branch": "master"
}
]
}

@ -0,0 +1,20 @@
Copyright © 2013 Keith Rarick
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,22 @@
Fernet takes a user-provided *message* (an arbitrary sequence of
bytes), a *key* (256 bits), and the current time, and produces a
*token*, which contains the message in a form that can't be read
or altered without the key.
This package is compatible with the other implementations at
https://github.com/fernet. They can exchange tokens freely among
each other.
Documentation: http://godoc.org/github.com/fernet/fernet-go
INSTALL
$ go get github.com/fernet/fernet-go
For more information and background, see the Fernet spec at
https://github.com/fernet/spec.
Fernet is distributed under the terms of the MIT license.
See the License file for details.

@ -0,0 +1,19 @@
package main
import (
"fmt"
"log"
"github.com/fernet/fernet-go"
)
func main() {
log.SetFlags(0)
log.SetPrefix("fernet: ")
var key fernet.Key
if err := key.Generate(); err != nil {
log.Fatal(err)
}
fmt.Println(key.Encode())
}

@ -0,0 +1,45 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"github.com/fernet/fernet-go"
)
const Usage = `Usage: fernet-sign ENV
fernet-sign encrypts and signs its input and prints the resulting token.
It uses the key in environment variable ENV.`
func main() {
log.SetFlags(0)
log.SetPrefix("fernet: ")
if len(os.Args) != 2 {
fmt.Fprintln(os.Stderr, Usage)
os.Exit(2)
}
key, err := fernet.DecodeKey(os.Getenv(os.Args[1]))
if err != nil {
log.Fatalln(err)
}
b, err := ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatalln(err)
}
t, err := fernet.EncryptAndSign(b, key)
if err != nil {
log.Fatalln(err)
}
_, err = os.Stdout.Write(append(t, '\n'))
if err != nil {
log.Fatalln(err)
}
}

@ -0,0 +1,168 @@
// Package fernet takes a user-provided message (an arbitrary
// sequence of bytes), a key (256 bits), and the current time,
// and produces a token, which contains the message in a form
// that can't be read or altered without the key.
//
// For more information and background, see the Fernet spec
// at https://github.com/fernet/spec.
//
// Subdirectories in this package provide command-line tools
// for working with Fernet keys and tokens.
package fernet
import (
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/binary"
"io"
"time"
)
const (
version byte = 0x80
tsOffset = 1
ivOffset = tsOffset + 8
payOffset = ivOffset + aes.BlockSize
overhead = 1 + 8 + aes.BlockSize + sha256.Size // ver + ts + iv + hmac
maxClockSkew = 60 * time.Second
)
var encoding = base64.URLEncoding
// generates a token from msg, writes it into tok, and returns the
// number of bytes generated, which is encodedLen(msg).
// len(tok) must be >= encodedLen(len(msg))
func gen(tok, msg, iv []byte, ts time.Time, k *Key) int {
tok[0] = version
binary.BigEndian.PutUint64(tok[tsOffset:], uint64(ts.Unix()))
copy(tok[ivOffset:], iv)
p := tok[payOffset:]
n := pad(p, msg, aes.BlockSize)
bc, _ := aes.NewCipher(k.cryptBytes())
cipher.NewCBCEncrypter(bc, iv).CryptBlocks(p[:n], p[:n])
genhmac(p[n:n], tok[:payOffset+n], k.signBytes())
return payOffset + n + sha256.Size
}
// token length for input msg of length n, not including base64
func encodedLen(n int) int {
const k = aes.BlockSize
return n/k*k + k + overhead
}
// max msg length for tok of length n, for binary token (no base64)
// upper bound; not exact
func decodedLen(n int) int {
return n - overhead
}
// if msg is nil, decrypts in place and returns a slice of tok.
func verify(msg, tok []byte, ttl time.Duration, now time.Time, k *Key) []byte {
if len(tok) < 1 || tok[0] != version {
return nil
}
ts := time.Unix(int64(binary.BigEndian.Uint64(tok[1:])), 0)
if ttl >= 0 && (now.After(ts.Add(ttl)) || ts.After(now.Add(maxClockSkew))) {
return nil
}
n := len(tok) - sha256.Size
var hmac [sha256.Size]byte
genhmac(hmac[:0], tok[:n], k.signBytes())
if subtle.ConstantTimeCompare(tok[n:], hmac[:]) != 1 {
return nil
}
pay := tok[payOffset : len(tok)-sha256.Size]
if len(pay)%aes.BlockSize != 0 {
return nil
}
if msg != nil {
copy(msg, pay)
pay = msg
}
bc, _ := aes.NewCipher(k.cryptBytes())
iv := tok[9:][:aes.BlockSize]
cipher.NewCBCDecrypter(bc, iv).CryptBlocks(pay, pay)
return unpad(pay)
}
// Pads p to a multiple of k using PKCS #7 standard block padding.
// See http://tools.ietf.org/html/rfc5652#section-6.3.
func pad(q, p []byte, k int) int {
n := len(p)/k*k + k
copy(q, p)
c := byte(n - len(p))
for i := len(p); i < n; i++ {
q[i] = c
}
return n
}
// Removes PKCS #7 standard block padding from p.
// See http://tools.ietf.org/html/rfc5652#section-6.3.
// This function is the inverse of pad.
// If the padding is not well-formed, unpad returns nil.
func unpad(p []byte) []byte {
c := p[len(p)-1]
for i := len(p) - int(c); i < len(p); i++ {
if i < 0 || p[i] != c {
return nil
}
}
return p[:len(p)-int(c)]
}
func b64enc(src []byte) []byte {
dst := make([]byte, encoding.EncodedLen(len(src)))
encoding.Encode(dst, src)
return dst
}
func b64dec(src []byte) []byte {
dst := make([]byte, encoding.DecodedLen(len(src)))
n, err := encoding.Decode(dst, src)
if err != nil {
return nil
}
return dst[:n]
}
func genhmac(q, p, k []byte) {
h := hmac.New(sha256.New, k)
h.Write(p)
h.Sum(q)
}
// EncryptAndSign encrypts and signs msg with key k and returns the resulting
// fernet token. If msg contains text, the text should be encoded
// with UTF-8 to follow fernet convention.
func EncryptAndSign(msg []byte, k *Key) (tok []byte, err error) {
iv := make([]byte, aes.BlockSize)
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
b := make([]byte, encodedLen(len(msg)))
n := gen(b, msg, iv, time.Now(), k)
tok = make([]byte, encoding.EncodedLen(n))
encoding.Encode(tok, b[:n])
return tok, nil
}
// VerifyAndDecrypt verifies that tok is a valid fernet token that was signed
// with a key in k at most ttl time ago only if ttl is greater than zero.
// Returns the message contained in tok if tok is valid, otherwise nil.
func VerifyAndDecrypt(tok []byte, ttl time.Duration, k []*Key) (msg []byte) {
b := make([]byte, encoding.DecodedLen(len(tok)))
n, _ := encoding.Decode(b, tok)
for _, k1 := range k {
msg = verify(nil, b[:n], ttl, time.Now(), k1)
if msg != nil {
return msg
}
}
return nil
}

@ -0,0 +1,9 @@
[
{
"token": "gAAAAAAdwJ6wAAECAwQFBgcICQoLDA0ODy021cpGVWKZ_eEwCGM4BLLF_5CV9dOPmrhuVUPgJobwOz7JcbmrR64jVmpU4IwqDA==",
"now": "1985-10-26T01:20:00-07:00",
"iv": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
"src": "hello",
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
}
]

@ -0,0 +1,58 @@
[
{
"desc": "incorrect mac",
"token": "gAAAAAAdwJ6xAAECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPAl1-szkFVzXTuGb4hR8AKtwcaX1YdykQUFBQUFBQUFBQQ==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "too short",
"token": "gAAAAAAdwJ6xAAECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPA==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "invalid base64",
"token": "%%%%%%%%%%%%%AECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPAl1-szkFVzXTuGb4hR8AKtwcaX1YdykRtfsH-p1YsUD2Q==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "payload size not multiple of block size",
"token": "gAAAAAAdwJ6xAAECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPOm73QeoCk9uGib28Xe5vz6oxq5nmxbx_v7mrfyudzUm",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "payload padding error",
"token": "gAAAAAAdwJ6xAAECAwQFBgcICQoLDA0ODz4LEpdELGQAad7aNEHbf-JkLPIpuiYRLQ3RtXatOYREu2FWke6CnJNYIbkuKNqOhw==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "far-future TS (unacceptable clock skew)",
"token": "gAAAAAAdwStRAAECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPAnja1xKYyhd-Y6mSkTOyTGJmw2Xc2a6kBd-iX9b_qXQcw==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "expired TTL",
"token": "gAAAAAAdwJ6xAAECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPAl1-szkFVzXTuGb4hR8AKtwcaX1YdykRtfsH-p1YsUD2Q==",
"now": "1985-10-26T01:21:31-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"desc": "incorrect IV (causes padding error)",
"token": "gAAAAAAdwJ6xBQECAwQFBgcICQoLDA0OD3HkMATM5lFqGaerZ-fWPAkLhFLHpGtDBRLRTZeUfWgHSv49TF2AUEZ1TIvcZjK1zQ==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
}
]

@ -0,0 +1,91 @@
package fernet
import (
"crypto/rand"
"encoding/base64"
"encoding/hex"
"errors"
"io"
)
var (
errKeyLen = errors.New("fernet: key decodes to wrong size")
errNoKeys = errors.New("fernet: no keys provided")
)
// Key represents a key.
type Key [32]byte
func (k *Key) cryptBytes() []byte {
return k[len(k)/2:]
}
func (k *Key) signBytes() []byte {
return k[:len(k)/2]
}
// Generate initializes k with pseudorandom data from package crypto/rand.
func (k *Key) Generate() error {
_, err := io.ReadFull(rand.Reader, k[:])
return err
}
// Encode returns the URL-safe base64 encoding of k.
func (k *Key) Encode() string {
return encoding.EncodeToString(k[:])
}
// DecodeKey decodes a key from s and returns it. The key can be in
// hexadecimal, standard base64, or URL-safe base64.
func DecodeKey(s string) (*Key, error) {
var b []byte
var err error
if s == "" {
return nil, errors.New("empty key")
}
if len(s) == hex.EncodedLen(len(Key{})) {
b, err = hex.DecodeString(s)
} else {
b, err = base64.StdEncoding.DecodeString(s)
if err != nil {
b, err = base64.URLEncoding.DecodeString(s)
}
}
if err != nil {
return nil, err
}
if len(b) != len(Key{}) {
return nil, errKeyLen
}
k := new(Key)
copy(k[:], b)
return k, nil
}
// DecodeKeys decodes each element of a using DecodeKey and returns the
// resulting keys. Requires at least one key.
func DecodeKeys(a ...string) ([]*Key, error) {
if len(a) == 0 {
return nil, errNoKeys
}
var err error
ks := make([]*Key, len(a))
for i, s := range a {
ks[i], err = DecodeKey(s)
if err != nil {
return nil, err
}
}
return ks, nil
}
// MustDecodeKeys is like DecodeKeys, but panics if an error occurs.
// It simplifies safe initialization of global variables holding
// keys.
func MustDecodeKeys(a ...string) []*Key {
k, err := DecodeKeys(a...)
if err != nil {
panic(err)
}
return k
}

@ -0,0 +1,16 @@
[
{
"token": "gAAAAAAdwJ6wAAECAwQFBgcICQoLDA0ODy021cpGVWKZ_eEwCGM4BLLF_5CV9dOPmrhuVUPgJobwOz7JcbmrR64jVmpU4IwqDA==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": 60,
"src": "hello",
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
},
{
"token": "gAAAAAAdwJ6wAAECAwQFBgcICQoLDA0ODy021cpGVWKZ_eEwCGM4BLLF_5CV9dOPmrhuVUPgJobwOz7JcbmrR64jVmpU4IwqDA==",
"now": "1985-10-26T01:20:01-07:00",
"ttl_sec": -1,
"src": "hello",
"secret": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4="
}
]

@ -0,0 +1,31 @@
Go support for Protocol Buffers - Google's data interchange format
Copyright 2010 The Go Authors. All rights reserved.
https://github.com/golang/protobuf
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

@ -0,0 +1,43 @@
# Go support for Protocol Buffers - Google's data interchange format
#
# Copyright 2010 The Go Authors. All rights reserved.
# https://github.com/golang/protobuf
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
install:
go install
test: install generate-test-pbs
go test
generate-test-pbs:
make install
make -C testdata
protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata:. proto3_proto/proto3.proto
make

@ -0,0 +1,223 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2011 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Protocol buffer deep copy and merge.
// TODO: RawMessage.
package proto
import (
"log"
"reflect"
"strings"
)
// Clone returns a deep copy of a protocol buffer.
func Clone(pb Message) Message {
in := reflect.ValueOf(pb)
if in.IsNil() {
return pb
}
out := reflect.New(in.Type().Elem())
// out is empty so a merge is a deep copy.
mergeStruct(out.Elem(), in.Elem())
return out.Interface().(Message)
}
// Merge merges src into dst.
// Required and optional fields that are set in src will be set to that value in dst.
// Elements of repeated fields will be appended.
// Merge panics if src and dst are not the same type, or if dst is nil.
func Merge(dst, src Message) {
in := reflect.ValueOf(src)
out := reflect.ValueOf(dst)
if out.IsNil() {
panic("proto: nil destination")
}
if in.Type() != out.Type() {
// Explicit test prior to mergeStruct so that mistyped nils will fail
panic("proto: type mismatch")
}
if in.IsNil() {
// Merging nil into non-nil is a quiet no-op
return
}
mergeStruct(out.Elem(), in.Elem())
}
func mergeStruct(out, in reflect.Value) {
sprop := GetProperties(in.Type())
for i := 0; i < in.NumField(); i++ {
f := in.Type().Field(i)
if strings.HasPrefix(f.Name, "XXX_") {
continue
}
mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
}
if emIn, ok := in.Addr().Interface().(extendableProto); ok {
emOut := out.Addr().Interface().(extendableProto)
mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap())
}
uf := in.FieldByName("XXX_unrecognized")
if !uf.IsValid() {
return
}
uin := uf.Bytes()
if len(uin) > 0 {
out.FieldByName("XXX_unrecognized").SetBytes(append([]byte(nil), uin...))
}
}
// mergeAny performs a merge between two values of the same type.
// viaPtr indicates whether the values were indirected through a pointer (implying proto2).
// prop is set if this is a struct field (it may be nil).
func mergeAny(out, in reflect.Value, viaPtr bool, prop *Properties) {
if in.Type() == protoMessageType {
if !in.IsNil() {
if out.IsNil() {
out.Set(reflect.ValueOf(Clone(in.Interface().(Message))))
} else {
Merge(out.Interface().(Message), in.Interface().(Message))
}
}
return
}
switch in.Kind() {
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
reflect.String, reflect.Uint32, reflect.Uint64:
if !viaPtr && isProto3Zero(in) {
return
}
out.Set(in)
case reflect.Interface:
// Probably a oneof field; copy non-nil values.
if in.IsNil() {
return
}
// Allocate destination if it is not set, or set to a different type.
// Otherwise we will merge as normal.
if out.IsNil() || out.Elem().Type() != in.Elem().Type() {
out.Set(reflect.New(in.Elem().Elem().Type())) // interface -> *T -> T -> new(T)
}
mergeAny(out.Elem(), in.Elem(), false, nil)
case reflect.Map:
if in.Len() == 0 {
return
}
if out.IsNil() {
out.Set(reflect.MakeMap(in.Type()))
}
// For maps with value types of *T or []byte we need to deep copy each value.
elemKind := in.Type().Elem().Kind()
for _, key := range in.MapKeys() {
var val reflect.Value
switch elemKind {
case reflect.Ptr:
val = reflect.New(in.Type().Elem().Elem())
mergeAny(val, in.MapIndex(key), false, nil)
case reflect.Slice:
val = in.MapIndex(key)
val = reflect.ValueOf(append([]byte{}, val.Bytes()...))
default:
val = in.MapIndex(key)
}
out.SetMapIndex(key, val)
}
case reflect.Ptr:
if in.IsNil() {
return
}
if out.IsNil() {
out.Set(reflect.New(in.Elem().Type()))
}
mergeAny(out.Elem(), in.Elem(), true, nil)
case reflect.Slice:
if in.IsNil() {
return
}
if in.Type().Elem().Kind() == reflect.Uint8 {
// []byte is a scalar bytes field, not a repeated field.
// Edge case: if this is in a proto3 message, a zero length
// bytes field is considered the zero value, and should not
// be merged.
if prop != nil && prop.proto3 && in.Len() == 0 {
return
}
// Make a deep copy.
// Append to []byte{} instead of []byte(nil) so that we never end up
// with a nil result.
out.SetBytes(append([]byte{}, in.Bytes()...))
return
}
n := in.Len()
if out.IsNil() {
out.Set(reflect.MakeSlice(in.Type(), 0, n))
}
switch in.Type().Elem().Kind() {
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
reflect.String, reflect.Uint32, reflect.Uint64:
out.Set(reflect.AppendSlice(out, in))
default:
for i := 0; i < n; i++ {
x := reflect.Indirect(reflect.New(in.Type().Elem()))
mergeAny(x, in.Index(i), false, nil)
out.Set(reflect.Append(out, x))
}
}
case reflect.Struct:
mergeStruct(out, in)
default:
// unknown type, so not a protocol buffer
log.Printf("proto: don't know how to copy %v", in)
}
}
func mergeExtension(out, in map[int32]Extension) {
for extNum, eIn := range in {
eOut := Extension{desc: eIn.desc}
if eIn.value != nil {
v := reflect.New(reflect.TypeOf(eIn.value)).Elem()
mergeAny(v, reflect.ValueOf(eIn.value), false, nil)
eOut.value = v.Interface()
}
if eIn.enc != nil {
eOut.enc = make([]byte, len(eIn.enc))
copy(eOut.enc, eIn.enc)
}
out[extNum] = eOut
}
}

@ -0,0 +1,867 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package proto
/*
* Routines for decoding protocol buffer data to construct in-memory representations.
*/
import (
"errors"
"fmt"
"io"
"os"
"reflect"
)
// errOverflow is returned when an integer is too large to be represented.
var errOverflow = errors.New("proto: integer overflow")
// ErrInternalBadWireType is returned by generated code when an incorrect
// wire type is encountered. It does not get returned to user code.
var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
// The fundamental decoders that interpret bytes on the wire.
// Those that take integer types all return uint64 and are
// therefore of type valueDecoder.
// DecodeVarint reads a varint-encoded integer from the slice.
// It returns the integer and the number of bytes consumed, or
// zero if there is not enough.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func DecodeVarint(buf []byte) (x uint64, n int) {
// x, n already 0
for shift := uint(0); shift < 64; shift += 7 {
if n >= len(buf) {
return 0, 0
}
b := uint64(buf[n])
n++
x |= (b & 0x7F) << shift
if (b & 0x80) == 0 {
return x, n
}
}
// The number is too large to represent in a 64-bit value.
return 0, 0
}
// DecodeVarint reads a varint-encoded integer from the Buffer.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func (p *Buffer) DecodeVarint() (x uint64, err error) {
// x, err already 0
i := p.index
l := len(p.buf)
for shift := uint(0); shift < 64; shift += 7 {
if i >= l {
err = io.ErrUnexpectedEOF
return
}
b := p.buf[i]
i++
x |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
p.index = i
return
}
}
// The number is too large to represent in a 64-bit value.
err = errOverflow
return
}
// DecodeFixed64 reads a 64-bit integer from the Buffer.
// This is the format for the
// fixed64, sfixed64, and double protocol buffer types.
func (p *Buffer) DecodeFixed64() (x uint64, err error) {
// x, err already 0
i := p.index + 8
if i < 0 || i > len(p.buf) {
err = io.ErrUnexpectedEOF
return
}
p.index = i
x = uint64(p.buf[i-8])
x |= uint64(p.buf[i-7]) << 8
x |= uint64(p.buf[i-6]) << 16
x |= uint64(p.buf[i-5]) << 24
x |= uint64(p.buf[i-4]) << 32
x |= uint64(p.buf[i-3]) << 40
x |= uint64(p.buf[i-2]) << 48
x |= uint64(p.buf[i-1]) << 56
return
}
// DecodeFixed32 reads a 32-bit integer from the Buffer.
// This is the format for the
// fixed32, sfixed32, and float protocol buffer types.
func (p *Buffer) DecodeFixed32() (x uint64, err error) {
// x, err already 0
i := p.index + 4
if i < 0 || i > len(p.buf) {
err = io.ErrUnexpectedEOF
return
}
p.index = i
x = uint64(p.buf[i-4])
x |= uint64(p.buf[i-3]) << 8
x |= uint64(p.buf[i-2]) << 16
x |= uint64(p.buf[i-1]) << 24
return
}
// DecodeZigzag64 reads a zigzag-encoded 64-bit integer
// from the Buffer.
// This is the format used for the sint64 protocol buffer type.
func (p *Buffer) DecodeZigzag64() (x uint64, err error) {
x, err = p.DecodeVarint()
if err != nil {
return
}
x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63)
return
}
// DecodeZigzag32 reads a zigzag-encoded 32-bit integer
// from the Buffer.
// This is the format used for the sint32 protocol buffer type.
func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
x, err = p.DecodeVarint()
if err != nil {
return
}
x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31))
return
}
// These are not ValueDecoders: they produce an array of bytes or a string.
// bytes, embedded messages
// DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
// This is the format used for the bytes protocol buffer
// type and for embedded messages.
func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) {
n, err := p.DecodeVarint()
if err != nil {
return nil, err
}
nb := int(n)
if nb < 0 {
return nil, fmt.Errorf("proto: bad byte length %d", nb)
}
end := p.index + nb
if end < p.index || end > len(p.buf) {
return nil, io.ErrUnexpectedEOF
}
if !alloc {
// todo: check if can get more uses of alloc=false
buf = p.buf[p.index:end]
p.index += nb
return
}
buf = make([]byte, nb)
copy(buf, p.buf[p.index:])
p.index += nb
return
}
// DecodeStringBytes reads an encoded string from the Buffer.
// This is the format used for the proto2 string type.
func (p *Buffer) DecodeStringBytes() (s string, err error) {
buf, err := p.DecodeRawBytes(false)
if err != nil {
return
}
return string(buf), nil
}
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
// If the protocol buffer has extensions, and the field matches, add it as an extension.
// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
oi := o.index
err := o.skip(t, tag, wire)
if err != nil {
return err
}
if !unrecField.IsValid() {
return nil
}
ptr := structPointer_Bytes(base, unrecField)
// Add the skipped field to struct field
obuf := o.buf
o.buf = *ptr
o.EncodeVarint(uint64(tag<<3 | wire))
*ptr = append(o.buf, obuf[oi:o.index]...)
o.buf = obuf
return nil
}
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
func (o *Buffer) skip(t reflect.Type, tag, wire int) error {
var u uint64
var err error
switch wire {
case WireVarint:
_, err = o.DecodeVarint()
case WireFixed64:
_, err = o.DecodeFixed64()
case WireBytes:
_, err = o.DecodeRawBytes(false)
case WireFixed32:
_, err = o.DecodeFixed32()
case WireStartGroup:
for {
u, err = o.DecodeVarint()
if err != nil {
break
}
fwire := int(u & 0x7)
if fwire == WireEndGroup {
break
}
ftag := int(u >> 3)
err = o.skip(t, ftag, fwire)
if err != nil {
break
}
}
default:
err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
}
return err
}
// Unmarshaler is the interface representing objects that can
// unmarshal themselves. The method should reset the receiver before
// decoding starts. The argument points to data that may be
// overwritten, so implementations should not keep references to the
// buffer.
type Unmarshaler interface {
Unmarshal([]byte) error
}
// Unmarshal parses the protocol buffer representation in buf and places the
// decoded result in pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable.
//
// Unmarshal resets pb before starting to unmarshal, so any
// existing data in pb is always removed. Use UnmarshalMerge
// to preserve and append to existing data.
func Unmarshal(buf []byte, pb Message) error {
pb.Reset()
return UnmarshalMerge(buf, pb)
}
// UnmarshalMerge parses the protocol buffer representation in buf and
// writes the decoded result to pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable.
//
// UnmarshalMerge merges into existing data in pb.
// Most code should use Unmarshal instead.
func UnmarshalMerge(buf []byte, pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(Unmarshaler); ok {
return u.Unmarshal(buf)
}
return NewBuffer(buf).Unmarshal(pb)
}
// DecodeMessage reads a count-delimited message from the Buffer.
func (p *Buffer) DecodeMessage(pb Message) error {
enc, err := p.DecodeRawBytes(false)
if err != nil {
return err
}
return NewBuffer(enc).Unmarshal(pb)
}
// DecodeGroup reads a tag-delimited group from the Buffer.
func (p *Buffer) DecodeGroup(pb Message) error {
typ, base, err := getbase(pb)
if err != nil {
return err
}
return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base)
}
// Unmarshal parses the protocol buffer representation in the
// Buffer and places the decoded result in pb. If the struct
// underlying pb does not match the data in the buffer, the results can be
// unpredictable.
func (p *Buffer) Unmarshal(pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(Unmarshaler); ok {
err := u.Unmarshal(p.buf[p.index:])
p.index = len(p.buf)
return err
}
typ, base, err := getbase(pb)
if err != nil {
return err
}
err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base)
if collectStats {
stats.Decode++
}
return err
}
// unmarshalType does the work of unmarshaling a structure.
func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error {
var state errorState
required, reqFields := prop.reqCount, uint64(0)
var err error
for err == nil && o.index < len(o.buf) {
oi := o.index
var u uint64
u, err = o.DecodeVarint()
if err != nil {
break
}
wire := int(u & 0x7)
if wire == WireEndGroup {
if is_group {
return nil // input is satisfied
}
return fmt.Errorf("proto: %s: wiretype end group for non-group", st)
}
tag := int(u >> 3)
if tag <= 0 {
return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire)
}
fieldnum, ok := prop.decoderTags.get(tag)
if !ok {
// Maybe it's an extension?
if prop.extendable {
if e := structPointer_Interface(base, st).(extendableProto); isExtensionField(e, int32(tag)) {
if err = o.skip(st, tag, wire); err == nil {
ext := e.ExtensionMap()[int32(tag)] // may be missing
ext.enc = append(ext.enc, o.buf[oi:o.index]...)
e.ExtensionMap()[int32(tag)] = ext
}
continue
}
}
// Maybe it's a oneof?
if prop.oneofUnmarshaler != nil {
m := structPointer_Interface(base, st).(Message)
// First return value indicates whether tag is a oneof field.
ok, err = prop.oneofUnmarshaler(m, tag, wire, o)
if err == ErrInternalBadWireType {
// Map the error to something more descriptive.
// Do the formatting here to save generated code space.
err = fmt.Errorf("bad wiretype for oneof field in %T", m)
}
if ok {
continue
}
}
err = o.skipAndSave(st, tag, wire, base, prop.unrecField)
continue
}
p := prop.Prop[fieldnum]
if p.dec == nil {
fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name)
continue
}
dec := p.dec
if wire != WireStartGroup && wire != p.WireType {
if wire == WireBytes && p.packedDec != nil {
// a packable field
dec = p.packedDec
} else {
err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType)
continue
}
}
decErr := dec(o, p, base)
if decErr != nil && !state.shouldContinue(decErr, p) {
err = decErr
}
if err == nil && p.Required {
// Successfully decoded a required field.
if tag <= 64 {
// use bitmap for fields 1-64 to catch field reuse.
var mask uint64 = 1 << uint64(tag-1)
if reqFields&mask == 0 {
// new required field
reqFields |= mask
required--
}
} else {
// This is imprecise. It can be fooled by a required field
// with a tag > 64 that is encoded twice; that's very rare.
// A fully correct implementation would require allocating
// a data structure, which we would like to avoid.
required--
}
}
}
if err == nil {
if is_group {
return io.ErrUnexpectedEOF
}
if state.err != nil {
return state.err
}
if required > 0 {
// Not enough information to determine the exact field. If we use extra
// CPU, we could determine the field only if the missing required field
// has a tag <= 64 and we check reqFields.
return &RequiredNotSetError{"{Unknown}"}
}
}
return err
}
// Individual type decoders
// For each,
// u is the decoded value,
// v is a pointer to the field (pointer) in the struct
// Sizes of the pools to allocate inside the Buffer.
// The goal is modest amortization and allocation
// on at least 16-byte boundaries.
const (
boolPoolSize = 16
uint32PoolSize = 8
uint64PoolSize = 4
)
// Decode a bool.
func (o *Buffer) dec_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
if len(o.bools) == 0 {
o.bools = make([]bool, boolPoolSize)
}
o.bools[0] = u != 0
*structPointer_Bool(base, p.field) = &o.bools[0]
o.bools = o.bools[1:]
return nil
}
func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
*structPointer_BoolVal(base, p.field) = u != 0
return nil
}
// Decode an int32.
func (o *Buffer) dec_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word32_Set(structPointer_Word32(base, p.field), o, uint32(u))
return nil
}
func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u))
return nil
}
// Decode an int64.
func (o *Buffer) dec_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word64_Set(structPointer_Word64(base, p.field), o, u)
return nil
}
func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word64Val_Set(structPointer_Word64Val(base, p.field), o, u)
return nil
}
// Decode a string.
func (o *Buffer) dec_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
*structPointer_String(base, p.field) = &s
return nil
}
func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
*structPointer_StringVal(base, p.field) = s
return nil
}
// Decode a slice of bytes ([]byte).
func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error {
b, err := o.DecodeRawBytes(true)
if err != nil {
return err
}
*structPointer_Bytes(base, p.field) = b
return nil
}
// Decode a slice of bools ([]bool).
func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
v := structPointer_BoolSlice(base, p.field)
*v = append(*v, u != 0)
return nil
}
// Decode a slice of bools ([]bool) in packed format.
func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error {
v := structPointer_BoolSlice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded bools
fin := o.index + nb
if fin < o.index {
return errOverflow
}
y := *v
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
y = append(y, u != 0)
}
*v = y
return nil
}
// Decode a slice of int32s ([]int32).
func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
structPointer_Word32Slice(base, p.field).Append(uint32(u))
return nil
}
// Decode a slice of int32s ([]int32) in packed format.
func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error {
v := structPointer_Word32Slice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded int32s
fin := o.index + nb
if fin < o.index {
return errOverflow
}
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
v.Append(uint32(u))
}
return nil
}
// Decode a slice of int64s ([]int64).
func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
structPointer_Word64Slice(base, p.field).Append(u)
return nil
}
// Decode a slice of int64s ([]int64) in packed format.
func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error {
v := structPointer_Word64Slice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded int64s
fin := o.index + nb
if fin < o.index {
return errOverflow
}
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
v.Append(u)
}
return nil
}
// Decode a slice of strings ([]string).
func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
v := structPointer_StringSlice(base, p.field)
*v = append(*v, s)
return nil
}
// Decode a slice of slice of bytes ([][]byte).
func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error {
b, err := o.DecodeRawBytes(true)
if err != nil {
return err
}
v := structPointer_BytesSlice(base, p.field)
*v = append(*v, b)
return nil
}
// Decode a map field.
func (o *Buffer) dec_new_map(p *Properties, base structPointer) error {
raw, err := o.DecodeRawBytes(false)
if err != nil {
return err
}
oi := o.index // index at the end of this map entry
o.index -= len(raw) // move buffer back to start of map entry
mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V
if mptr.Elem().IsNil() {
mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem()))
}
v := mptr.Elem() // map[K]V
// Prepare addressable doubly-indirect placeholders for the key and value types.
// See enc_new_map for why.
keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K
keybase := toStructPointer(keyptr.Addr()) // **K
var valbase structPointer
var valptr reflect.Value
switch p.mtype.Elem().Kind() {
case reflect.Slice:
// []byte
var dummy []byte
valptr = reflect.ValueOf(&dummy) // *[]byte
valbase = toStructPointer(valptr) // *[]byte
case reflect.Ptr:
// message; valptr is **Msg; need to allocate the intermediate pointer
valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
valptr.Set(reflect.New(valptr.Type().Elem()))
valbase = toStructPointer(valptr)
default:
// everything else
valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
valbase = toStructPointer(valptr.Addr()) // **V
}
// Decode.
// This parses a restricted wire format, namely the encoding of a message
// with two fields. See enc_new_map for the format.
for o.index < oi {
// tagcode for key and value properties are always a single byte
// because they have tags 1 and 2.
tagcode := o.buf[o.index]
o.index++
switch tagcode {
case p.mkeyprop.tagcode[0]:
if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil {
return err
}
case p.mvalprop.tagcode[0]:
if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil {
return err
}
default:
// TODO: Should we silently skip this instead?
return fmt.Errorf("proto: bad map data tag %d", raw[0])
}
}
keyelem, valelem := keyptr.Elem(), valptr.Elem()
if !keyelem.IsValid() || !valelem.IsValid() {
// We did not decode the key or the value in the map entry.
// Either way, it's an invalid map entry.
return fmt.Errorf("proto: bad map data: missing key/val")
}
v.SetMapIndex(keyelem, valelem)
return nil
}
// Decode a group.
func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error {
bas := structPointer_GetStructPointer(base, p.field)
if structPointer_IsNil(bas) {
// allocate new nested message
bas = toStructPointer(reflect.New(p.stype))
structPointer_SetStructPointer(base, p.field, bas)
}
return o.unmarshalType(p.stype, p.sprop, true, bas)
}
// Decode an embedded message.
func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) {
raw, e := o.DecodeRawBytes(false)
if e != nil {
return e
}
bas := structPointer_GetStructPointer(base, p.field)
if structPointer_IsNil(bas) {
// allocate new nested message
bas = toStructPointer(reflect.New(p.stype))
structPointer_SetStructPointer(base, p.field, bas)
}
// If the object can unmarshal itself, let it.
if p.isUnmarshaler {
iv := structPointer_Interface(bas, p.stype)
return iv.(Unmarshaler).Unmarshal(raw)
}
obuf := o.buf
oi := o.index
o.buf = raw
o.index = 0
err = o.unmarshalType(p.stype, p.sprop, false, bas)
o.buf = obuf
o.index = oi
return err
}
// Decode a slice of embedded messages.
func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error {
return o.dec_slice_struct(p, false, base)
}
// Decode a slice of embedded groups.
func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error {
return o.dec_slice_struct(p, true, base)
}
// Decode a slice of structs ([]*struct).
func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error {
v := reflect.New(p.stype)
bas := toStructPointer(v)
structPointer_StructPointerSlice(base, p.field).Append(bas)
if is_group {
err := o.unmarshalType(p.stype, p.sprop, is_group, bas)
return err
}
raw, err := o.DecodeRawBytes(false)
if err != nil {
return err
}
// If the object can unmarshal itself, let it.
if p.isUnmarshaler {
iv := v.Interface()
return iv.(Unmarshaler).Unmarshal(raw)
}
obuf := o.buf
oi := o.index
o.buf = raw
o.index = 0
err = o.unmarshalType(p.stype, p.sprop, is_group, bas)
o.buf = obuf
o.index = oi
return err
}

File diff suppressed because it is too large Load Diff

@ -0,0 +1,276 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2011 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Protocol buffer comparison.
package proto
import (
"bytes"
"log"
"reflect"
"strings"
)
/*
Equal returns true iff protocol buffers a and b are equal.
The arguments must both be pointers to protocol buffer structs.
Equality is defined in this way:
- Two messages are equal iff they are the same type,
corresponding fields are equal, unknown field sets
are equal, and extensions sets are equal.
- Two set scalar fields are equal iff their values are equal.
If the fields are of a floating-point type, remember that
NaN != x for all x, including NaN. If the message is defined
in a proto3 .proto file, fields are not "set"; specifically,
zero length proto3 "bytes" fields are equal (nil == {}).
- Two repeated fields are equal iff their lengths are the same,
and their corresponding elements are equal (a "bytes" field,
although represented by []byte, is not a repeated field)
- Two unset fields are equal.
- Two unknown field sets are equal if their current
encoded state is equal.
- Two extension sets are equal iff they have corresponding
elements that are pairwise equal.
- Every other combination of things are not equal.
The return value is undefined if a and b are not protocol buffers.
*/
func Equal(a, b Message) bool {
if a == nil || b == nil {
return a == b
}
v1, v2 := reflect.ValueOf(a), reflect.ValueOf(b)
if v1.Type() != v2.Type() {
return false
}
if v1.Kind() == reflect.Ptr {
if v1.IsNil() {
return v2.IsNil()
}
if v2.IsNil() {
return false
}
v1, v2 = v1.Elem(), v2.Elem()
}
if v1.Kind() != reflect.Struct {
return false
}
return equalStruct(v1, v2)
}
// v1 and v2 are known to have the same type.
func equalStruct(v1, v2 reflect.Value) bool {
sprop := GetProperties(v1.Type())
for i := 0; i < v1.NumField(); i++ {
f := v1.Type().Field(i)
if strings.HasPrefix(f.Name, "XXX_") {
continue
}
f1, f2 := v1.Field(i), v2.Field(i)
if f.Type.Kind() == reflect.Ptr {
if n1, n2 := f1.IsNil(), f2.IsNil(); n1 && n2 {
// both unset
continue
} else if n1 != n2 {
// set/unset mismatch
return false
}
b1, ok := f1.Interface().(raw)
if ok {
b2 := f2.Interface().(raw)
// RawMessage
if !bytes.Equal(b1.Bytes(), b2.Bytes()) {
return false
}
continue
}
f1, f2 = f1.Elem(), f2.Elem()
}
if !equalAny(f1, f2, sprop.Prop[i]) {
return false
}
}
if em1 := v1.FieldByName("XXX_extensions"); em1.IsValid() {
em2 := v2.FieldByName("XXX_extensions")
if !equalExtensions(v1.Type(), em1.Interface().(map[int32]Extension), em2.Interface().(map[int32]Extension)) {
return false
}
}
uf := v1.FieldByName("XXX_unrecognized")
if !uf.IsValid() {
return true
}
u1 := uf.Bytes()
u2 := v2.FieldByName("XXX_unrecognized").Bytes()
if !bytes.Equal(u1, u2) {
return false
}
return true
}
// v1 and v2 are known to have the same type.
// prop may be nil.
func equalAny(v1, v2 reflect.Value, prop *Properties) bool {
if v1.Type() == protoMessageType {
m1, _ := v1.Interface().(Message)
m2, _ := v2.Interface().(Message)
return Equal(m1, m2)
}
switch v1.Kind() {
case reflect.Bool:
return v1.Bool() == v2.Bool()
case reflect.Float32, reflect.Float64:
return v1.Float() == v2.Float()
case reflect.Int32, reflect.Int64:
return v1.Int() == v2.Int()
case reflect.Interface:
// Probably a oneof field; compare the inner values.
n1, n2 := v1.IsNil(), v2.IsNil()
if n1 || n2 {
return n1 == n2
}
e1, e2 := v1.Elem(), v2.Elem()
if e1.Type() != e2.Type() {
return false
}
return equalAny(e1, e2, nil)
case reflect.Map:
if v1.Len() != v2.Len() {
return false
}
for _, key := range v1.MapKeys() {
val2 := v2.MapIndex(key)
if !val2.IsValid() {
// This key was not found in the second map.
return false
}
if !equalAny(v1.MapIndex(key), val2, nil) {
return false
}
}
return true
case reflect.Ptr:
return equalAny(v1.Elem(), v2.Elem(), prop)
case reflect.Slice:
if v1.Type().Elem().Kind() == reflect.Uint8 {
// short circuit: []byte
// Edge case: if this is in a proto3 message, a zero length
// bytes field is considered the zero value.
if prop != nil && prop.proto3 && v1.Len() == 0 && v2.Len() == 0 {
return true
}
if v1.IsNil() != v2.IsNil() {
return false
}
return bytes.Equal(v1.Interface().([]byte), v2.Interface().([]byte))
}
if v1.Len() != v2.Len() {
return false
}
for i := 0; i < v1.Len(); i++ {
if !equalAny(v1.Index(i), v2.Index(i), prop) {
return false
}
}
return true
case reflect.String:
return v1.Interface().(string) == v2.Interface().(string)
case reflect.Struct:
return equalStruct(v1, v2)
case reflect.Uint32, reflect.Uint64:
return v1.Uint() == v2.Uint()
}
// unknown type, so not a protocol buffer
log.Printf("proto: don't know how to compare %v", v1)
return false
}
// base is the struct type that the extensions are based on.
// em1 and em2 are extension maps.
func equalExtensions(base reflect.Type, em1, em2 map[int32]Extension) bool {
if len(em1) != len(em2) {
return false
}
for extNum, e1 := range em1 {
e2, ok := em2[extNum]
if !ok {
return false
}
m1, m2 := e1.value, e2.value
if m1 != nil && m2 != nil {
// Both are unencoded.
if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) {
return false
}
continue
}
// At least one is encoded. To do a semantically correct comparison
// we need to unmarshal them first.
var desc *ExtensionDesc
if m := extensionMaps[base]; m != nil {
desc = m[extNum]
}
if desc == nil {
log.Printf("proto: don't know how to compare extension %d of %v", extNum, base)
continue
}
var err error
if m1 == nil {
m1, err = decodeExtension(e1.enc, desc)
}
if m2 == nil && err == nil {
m2, err = decodeExtension(e2.enc, desc)
}
if err != nil {
// The encoded form is invalid.
log.Printf("proto: badly encoded extension %d of %v: %v", extNum, base, err)
return false
}
if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) {
return false
}
}
return true
}

@ -0,0 +1,399 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package proto
/*
* Types and routines for supporting protocol buffer extensions.
*/
import (
"errors"
"fmt"
"reflect"
"strconv"
"sync"
)
// ErrMissingExtension is the error returned by GetExtension if the named extension is not in the message.
var ErrMissingExtension = errors.New("proto: missing extension")
// ExtensionRange represents a range of message extensions for a protocol buffer.
// Used in code generated by the protocol compiler.
type ExtensionRange struct {
Start, End int32 // both inclusive
}
// extendableProto is an interface implemented by any protocol buffer that may be extended.
type extendableProto interface {
Message
ExtensionRangeArray() []ExtensionRange
ExtensionMap() map[int32]Extension
}
var extendableProtoType = reflect.TypeOf((*extendableProto)(nil)).Elem()
// ExtensionDesc represents an extension specification.
// Used in generated code from the protocol compiler.
type ExtensionDesc struct {
ExtendedType Message // nil pointer to the type that is being extended
ExtensionType interface{} // nil pointer to the extension type
Field int32 // field number
Name string // fully-qualified name of extension, for text formatting
Tag string // protobuf tag style
}
func (ed *ExtensionDesc) repeated() bool {
t := reflect.TypeOf(ed.ExtensionType)
return t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8
}
// Extension represents an extension in a message.
type Extension struct {
// When an extension is stored in a message using SetExtension
// only desc and value are set. When the message is marshaled
// enc will be set to the encoded form of the message.
//
// When a message is unmarshaled and contains extensions, each
// extension will have only enc set. When such an extension is
// accessed using GetExtension (or GetExtensions) desc and value
// will be set.
desc *ExtensionDesc
value interface{}
enc []byte
}
// SetRawExtension is for testing only.
func SetRawExtension(base extendableProto, id int32, b []byte) {
base.ExtensionMap()[id] = Extension{enc: b}
}
// isExtensionField returns true iff the given field number is in an extension range.
func isExtensionField(pb extendableProto, field int32) bool {
for _, er := range pb.ExtensionRangeArray() {
if er.Start <= field && field <= er.End {
return true
}
}
return false
}
// checkExtensionTypes checks that the given extension is valid for pb.
func checkExtensionTypes(pb extendableProto, extension *ExtensionDesc) error {
// Check the extended type.
if a, b := reflect.TypeOf(pb), reflect.TypeOf(extension.ExtendedType); a != b {
return errors.New("proto: bad extended type; " + b.String() + " does not extend " + a.String())
}
// Check the range.
if !isExtensionField(pb, extension.Field) {
return errors.New("proto: bad extension number; not in declared ranges")
}
return nil
}
// extPropKey is sufficient to uniquely identify an extension.
type extPropKey struct {
base reflect.Type
field int32
}
var extProp = struct {
sync.RWMutex
m map[extPropKey]*Properties
}{
m: make(map[extPropKey]*Properties),
}
func extensionProperties(ed *ExtensionDesc) *Properties {
key := extPropKey{base: reflect.TypeOf(ed.ExtendedType), field: ed.Field}
extProp.RLock()
if prop, ok := extProp.m[key]; ok {
extProp.RUnlock()
return prop
}
extProp.RUnlock()
extProp.Lock()
defer extProp.Unlock()
// Check again.
if prop, ok := extProp.m[key]; ok {
return prop
}
prop := new(Properties)
prop.Init(reflect.TypeOf(ed.ExtensionType), "unknown_name", ed.Tag, nil)
extProp.m[key] = prop
return prop
}
// encodeExtensionMap encodes any unmarshaled (unencoded) extensions in m.
func encodeExtensionMap(m map[int32]Extension) error {
for k, e := range m {
if e.value == nil || e.desc == nil {
// Extension is only in its encoded form.
continue
}
// We don't skip extensions that have an encoded form set,
// because the extension value may have been mutated after
// the last time this function was called.
et := reflect.TypeOf(e.desc.ExtensionType)
props := extensionProperties(e.desc)
p := NewBuffer(nil)
// If e.value has type T, the encoder expects a *struct{ X T }.
// Pass a *T with a zero field and hope it all works out.
x := reflect.New(et)
x.Elem().Set(reflect.ValueOf(e.value))
if err := props.enc(p, props, toStructPointer(x)); err != nil {
return err
}
e.enc = p.buf
m[k] = e
}
return nil
}
func sizeExtensionMap(m map[int32]Extension) (n int) {
for _, e := range m {
if e.value == nil || e.desc == nil {
// Extension is only in its encoded form.
n += len(e.enc)
continue
}
// We don't skip extensions that have an encoded form set,
// because the extension value may have been mutated after
// the last time this function was called.
et := reflect.TypeOf(e.desc.ExtensionType)
props := extensionProperties(e.desc)
// If e.value has type T, the encoder expects a *struct{ X T }.
// Pass a *T with a zero field and hope it all works out.
x := reflect.New(et)
x.Elem().Set(reflect.ValueOf(e.value))
n += props.size(props, toStructPointer(x))
}
return
}
// HasExtension returns whether the given extension is present in pb.
func HasExtension(pb extendableProto, extension *ExtensionDesc) bool {
// TODO: Check types, field numbers, etc.?
_, ok := pb.ExtensionMap()[extension.Field]
return ok
}
// ClearExtension removes the given extension from pb.
func ClearExtension(pb extendableProto, extension *ExtensionDesc) {
// TODO: Check types, field numbers, etc.?
delete(pb.ExtensionMap(), extension.Field)
}
// GetExtension parses and returns the given extension of pb.
// If the extension is not present and has no default value it returns ErrMissingExtension.
func GetExtension(pb extendableProto, extension *ExtensionDesc) (interface{}, error) {
if err := checkExtensionTypes(pb, extension); err != nil {
return nil, err
}
emap := pb.ExtensionMap()
e, ok := emap[extension.Field]
if !ok {
// defaultExtensionValue returns the default value or
// ErrMissingExtension if there is no default.
return defaultExtensionValue(extension)
}
if e.value != nil {
// Already decoded. Check the descriptor, though.
if e.desc != extension {
// This shouldn't happen. If it does, it means that
// GetExtension was called twice with two different
// descriptors with the same field number.
return nil, errors.New("proto: descriptor conflict")
}
return e.value, nil
}
v, err := decodeExtension(e.enc, extension)
if err != nil {
return nil, err
}
// Remember the decoded version and drop the encoded version.
// That way it is safe to mutate what we return.
e.value = v
e.desc = extension
e.enc = nil
emap[extension.Field] = e
return e.value, nil
}
// defaultExtensionValue returns the default value for extension.
// If no default for an extension is defined ErrMissingExtension is returned.
func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) {
t := reflect.TypeOf(extension.ExtensionType)
props := extensionProperties(extension)
sf, _, err := fieldDefault(t, props)
if err != nil {
return nil, err
}
if sf == nil || sf.value == nil {
// There is no default value.
return nil, ErrMissingExtension
}
if t.Kind() != reflect.Ptr {
// We do not need to return a Ptr, we can directly return sf.value.
return sf.value, nil
}
// We need to return an interface{} that is a pointer to sf.value.
value := reflect.New(t).Elem()
value.Set(reflect.New(value.Type().Elem()))
if sf.kind == reflect.Int32 {
// We may have an int32 or an enum, but the underlying data is int32.
// Since we can't set an int32 into a non int32 reflect.value directly
// set it as a int32.
value.Elem().SetInt(int64(sf.value.(int32)))
} else {
value.Elem().Set(reflect.ValueOf(sf.value))
}
return value.Interface(), nil
}
// decodeExtension decodes an extension encoded in b.
func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) {
o := NewBuffer(b)
t := reflect.TypeOf(extension.ExtensionType)
props := extensionProperties(extension)
// t is a pointer to a struct, pointer to basic type or a slice.
// Allocate a "field" to store the pointer/slice itself; the
// pointer/slice will be stored here. We pass
// the address of this field to props.dec.
// This passes a zero field and a *t and lets props.dec
// interpret it as a *struct{ x t }.
value := reflect.New(t).Elem()
for {
// Discard wire type and field number varint. It isn't needed.
if _, err := o.DecodeVarint(); err != nil {
return nil, err
}
if err := props.dec(o, props, toStructPointer(value.Addr())); err != nil {
return nil, err
}
if o.index >= len(o.buf) {
break
}
}
return value.Interface(), nil
}
// GetExtensions returns a slice of the extensions present in pb that are also listed in es.
// The returned slice has the same length as es; missing extensions will appear as nil elements.
func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) {
epb, ok := pb.(extendableProto)
if !ok {
err = errors.New("proto: not an extendable proto")
return
}
extensions = make([]interface{}, len(es))
for i, e := range es {
extensions[i], err = GetExtension(epb, e)
if err == ErrMissingExtension {
err = nil
}
if err != nil {
return
}
}
return
}
// SetExtension sets the specified extension of pb to the specified value.
func SetExtension(pb extendableProto, extension *ExtensionDesc, value interface{}) error {
if err := checkExtensionTypes(pb, extension); err != nil {
return err
}
typ := reflect.TypeOf(extension.ExtensionType)
if typ != reflect.TypeOf(value) {
return errors.New("proto: bad extension value type")
}
// nil extension values need to be caught early, because the
// encoder can't distinguish an ErrNil due to a nil extension
// from an ErrNil due to a missing field. Extensions are
// always optional, so the encoder would just swallow the error
// and drop all the extensions from the encoded message.
if reflect.ValueOf(value).IsNil() {
return fmt.Errorf("proto: SetExtension called with nil value of type %T", value)
}
pb.ExtensionMap()[extension.Field] = Extension{desc: extension, value: value}
return nil
}
// A global registry of extensions.
// The generated code will register the generated descriptors by calling RegisterExtension.
var extensionMaps = make(map[reflect.Type]map[int32]*ExtensionDesc)
// RegisterExtension is called from the generated code.
func RegisterExtension(desc *ExtensionDesc) {
st := reflect.TypeOf(desc.ExtendedType).Elem()
m := extensionMaps[st]
if m == nil {
m = make(map[int32]*ExtensionDesc)
extensionMaps[st] = m
}
if _, ok := m[desc.Field]; ok {
panic("proto: duplicate extension registered: " + st.String() + " " + strconv.Itoa(int(desc.Field)))
}
m[desc.Field] = desc
}
// RegisteredExtensions returns a map of the registered extensions of a
// protocol buffer struct, indexed by the extension number.
// The argument pb should be a nil pointer to the struct type.
func RegisteredExtensions(pb Message) map[int32]*ExtensionDesc {
return extensionMaps[reflect.TypeOf(pb).Elem()]
}

@ -0,0 +1,893 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/*
Package proto converts data structures to and from the wire format of
protocol buffers. It works in concert with the Go source code generated
for .proto files by the protocol compiler.
A summary of the properties of the protocol buffer interface
for a protocol buffer variable v:
- Names are turned from camel_case to CamelCase for export.
- There are no methods on v to set fields; just treat
them as structure fields.
- There are getters that return a field's value if set,
and return the field's default value if unset.
The getters work even if the receiver is a nil message.
- The zero value for a struct is its correct initialization state.
All desired fields must be set before marshaling.
- A Reset() method will restore a protobuf struct to its zero state.
- Non-repeated fields are pointers to the values; nil means unset.
That is, optional or required field int32 f becomes F *int32.
- Repeated fields are slices.
- Helper functions are available to aid the setting of fields.
msg.Foo = proto.String("hello") // set field
- Constants are defined to hold the default values of all fields that
have them. They have the form Default_StructName_FieldName.
Because the getter methods handle defaulted values,
direct use of these constants should be rare.
- Enums are given type names and maps from names to values.
Enum values are prefixed by the enclosing message's name, or by the
enum's type name if it is a top-level enum. Enum types have a String
method, and a Enum method to assist in message construction.
- Nested messages, groups and enums have type names prefixed with the name of
the surrounding message type.
- Extensions are given descriptor names that start with E_,
followed by an underscore-delimited list of the nested messages
that contain it (if any) followed by the CamelCased name of the
extension field itself. HasExtension, ClearExtension, GetExtension
and SetExtension are functions for manipulating extensions.
- Oneof field sets are given a single field in their message,
with distinguished wrapper types for each possible field value.
- Marshal and Unmarshal are functions to encode and decode the wire format.
When the .proto file specifies `syntax="proto3"`, there are some differences:
- Non-repeated fields of non-message type are values instead of pointers.
- Getters are only generated for message and oneof fields.
- Enum types do not get an Enum method.
The simplest way to describe this is to see an example.
Given file test.proto, containing
package example;
enum FOO { X = 17; }
message Test {
required string label = 1;
optional int32 type = 2 [default=77];
repeated int64 reps = 3;
optional group OptionalGroup = 4 {
required string RequiredField = 5;
}
oneof union {
int32 number = 6;
string name = 7;
}
}
The resulting file, test.pb.go, is:
package example
import proto "github.com/golang/protobuf/proto"
import math "math"
type FOO int32
const (
FOO_X FOO = 17
)
var FOO_name = map[int32]string{
17: "X",
}
var FOO_value = map[string]int32{
"X": 17,
}
func (x FOO) Enum() *FOO {
p := new(FOO)
*p = x
return p
}
func (x FOO) String() string {
return proto.EnumName(FOO_name, int32(x))
}
func (x *FOO) UnmarshalJSON(data []byte) error {
value, err := proto.UnmarshalJSONEnum(FOO_value, data)
if err != nil {
return err
}
*x = FOO(value)
return nil
}
type Test struct {
Label *string `protobuf:"bytes,1,req,name=label" json:"label,omitempty"`
Type *int32 `protobuf:"varint,2,opt,name=type,def=77" json:"type,omitempty"`
Reps []int64 `protobuf:"varint,3,rep,name=reps" json:"reps,omitempty"`
Optionalgroup *Test_OptionalGroup `protobuf:"group,4,opt,name=OptionalGroup" json:"optionalgroup,omitempty"`
// Types that are valid to be assigned to Union:
// *Test_Number
// *Test_Name
Union isTest_Union `protobuf_oneof:"union"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Test) Reset() { *m = Test{} }
func (m *Test) String() string { return proto.CompactTextString(m) }
func (*Test) ProtoMessage() {}
type isTest_Union interface {
isTest_Union()
}
type Test_Number struct {
Number int32 `protobuf:"varint,6,opt,name=number"`
}
type Test_Name struct {
Name string `protobuf:"bytes,7,opt,name=name"`
}
func (*Test_Number) isTest_Union() {}
func (*Test_Name) isTest_Union() {}
func (m *Test) GetUnion() isTest_Union {
if m != nil {
return m.Union
}
return nil
}
const Default_Test_Type int32 = 77
func (m *Test) GetLabel() string {
if m != nil && m.Label != nil {
return *m.Label
}
return ""
}
func (m *Test) GetType() int32 {
if m != nil && m.Type != nil {
return *m.Type
}
return Default_Test_Type
}
func (m *Test) GetOptionalgroup() *Test_OptionalGroup {
if m != nil {
return m.Optionalgroup
}
return nil
}
type Test_OptionalGroup struct {
RequiredField *string `protobuf:"bytes,5,req" json:"RequiredField,omitempty"`
}
func (m *Test_OptionalGroup) Reset() { *m = Test_OptionalGroup{} }
func (m *Test_OptionalGroup) String() string { return proto.CompactTextString(m) }
func (m *Test_OptionalGroup) GetRequiredField() string {
if m != nil && m.RequiredField != nil {
return *m.RequiredField
}
return ""
}
func (m *Test) GetNumber() int32 {
if x, ok := m.GetUnion().(*Test_Number); ok {
return x.Number
}
return 0
}
func (m *Test) GetName() string {
if x, ok := m.GetUnion().(*Test_Name); ok {
return x.Name
}
return ""
}
func init() {
proto.RegisterEnum("example.FOO", FOO_name, FOO_value)
}
To create and play with a Test object:
package main
import (
"log"
"github.com/golang/protobuf/proto"
pb "./example.pb"
)
func main() {
test := &pb.Test{
Label: proto.String("hello"),
Type: proto.Int32(17),
Optionalgroup: &pb.Test_OptionalGroup{
RequiredField: proto.String("good bye"),
},
Union: &pb.Test_Name{"fred"},
}
data, err := proto.Marshal(test)
if err != nil {
log.Fatal("marshaling error: ", err)
}
newTest := &pb.Test{}
err = proto.Unmarshal(data, newTest)
if err != nil {
log.Fatal("unmarshaling error: ", err)
}
// Now test and newTest contain the same data.
if test.GetLabel() != newTest.GetLabel() {
log.Fatalf("data mismatch %q != %q", test.GetLabel(), newTest.GetLabel())
}
// Use a type switch to determine which oneof was set.
switch u := test.Union.(type) {
case *pb.Test_Number: // u.Number contains the number.
case *pb.Test_Name: // u.Name contains the string.
}
// etc.
}
*/
package proto
import (
"encoding/json"
"fmt"
"log"
"reflect"
"sort"
"strconv"
"sync"
)
// Message is implemented by generated protocol buffer messages.
type Message interface {
Reset()
String() string
ProtoMessage()
}
// Stats records allocation details about the protocol buffer encoders
// and decoders. Useful for tuning the library itself.
type Stats struct {
Emalloc uint64 // mallocs in encode
Dmalloc uint64 // mallocs in decode
Encode uint64 // number of encodes
Decode uint64 // number of decodes
Chit uint64 // number of cache hits
Cmiss uint64 // number of cache misses
Size uint64 // number of sizes
}
// Set to true to enable stats collection.
const collectStats = false
var stats Stats
// GetStats returns a copy of the global Stats structure.
func GetStats() Stats { return stats }
// A Buffer is a buffer manager for marshaling and unmarshaling
// protocol buffers. It may be reused between invocations to
// reduce memory usage. It is not necessary to use a Buffer;
// the global functions Marshal and Unmarshal create a
// temporary Buffer and are fine for most applications.
type Buffer struct {
buf []byte // encode/decode byte stream
index int // write point
// pools of basic types to amortize allocation.
bools []bool
uint32s []uint32
uint64s []uint64
// extra pools, only used with pointer_reflect.go
int32s []int32
int64s []int64
float32s []float32
float64s []float64
}
// NewBuffer allocates a new Buffer and initializes its internal data to
// the contents of the argument slice.
func NewBuffer(e []byte) *Buffer {
return &Buffer{buf: e}
}
// Reset resets the Buffer, ready for marshaling a new protocol buffer.
func (p *Buffer) Reset() {
p.buf = p.buf[0:0] // for reading/writing
p.index = 0 // for reading
}
// SetBuf replaces the internal buffer with the slice,
// ready for unmarshaling the contents of the slice.
func (p *Buffer) SetBuf(s []byte) {
p.buf = s
p.index = 0
}
// Bytes returns the contents of the Buffer.
func (p *Buffer) Bytes() []byte { return p.buf }
/*
* Helper routines for simplifying the creation of optional fields of basic type.
*/
// Bool is a helper routine that allocates a new bool value
// to store v and returns a pointer to it.
func Bool(v bool) *bool {
return &v
}
// Int32 is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it.
func Int32(v int32) *int32 {
return &v
}
// Int is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it, but unlike Int32
// its argument value is an int.
func Int(v int) *int32 {
p := new(int32)
*p = int32(v)
return p
}
// Int64 is a helper routine that allocates a new int64 value
// to store v and returns a pointer to it.
func Int64(v int64) *int64 {
return &v
}
// Float32 is a helper routine that allocates a new float32 value
// to store v and returns a pointer to it.
func Float32(v float32) *float32 {
return &v
}
// Float64 is a helper routine that allocates a new float64 value
// to store v and returns a pointer to it.
func Float64(v float64) *float64 {
return &v
}
// Uint32 is a helper routine that allocates a new uint32 value
// to store v and returns a pointer to it.
func Uint32(v uint32) *uint32 {
return &v
}
// Uint64 is a helper routine that allocates a new uint64 value
// to store v and returns a pointer to it.
func Uint64(v uint64) *uint64 {
return &v
}
// String is a helper routine that allocates a new string value
// to store v and returns a pointer to it.
func String(v string) *string {
return &v
}
// EnumName is a helper function to simplify printing protocol buffer enums
// by name. Given an enum map and a value, it returns a useful string.
func EnumName(m map[int32]string, v int32) string {
s, ok := m[v]
if ok {
return s
}
return strconv.Itoa(int(v))
}
// UnmarshalJSONEnum is a helper function to simplify recovering enum int values
// from their JSON-encoded representation. Given a map from the enum's symbolic
// names to its int values, and a byte buffer containing the JSON-encoded
// value, it returns an int32 that can be cast to the enum type by the caller.
//
// The function can deal with both JSON representations, numeric and symbolic.
func UnmarshalJSONEnum(m map[string]int32, data []byte, enumName string) (int32, error) {
if data[0] == '"' {
// New style: enums are strings.
var repr string
if err := json.Unmarshal(data, &repr); err != nil {
return -1, err
}
val, ok := m[repr]
if !ok {
return 0, fmt.Errorf("unrecognized enum %s value %q", enumName, repr)
}
return val, nil
}
// Old style: enums are ints.
var val int32
if err := json.Unmarshal(data, &val); err != nil {
return 0, fmt.Errorf("cannot unmarshal %#q into enum %s", data, enumName)
}
return val, nil
}
// DebugPrint dumps the encoded data in b in a debugging format with a header
// including the string s. Used in testing but made available for general debugging.
func (p *Buffer) DebugPrint(s string, b []byte) {
var u uint64
obuf := p.buf
index := p.index
p.buf = b
p.index = 0
depth := 0
fmt.Printf("\n--- %s ---\n", s)
out:
for {
for i := 0; i < depth; i++ {
fmt.Print(" ")
}
index := p.index
if index == len(p.buf) {
break
}
op, err := p.DecodeVarint()
if err != nil {
fmt.Printf("%3d: fetching op err %v\n", index, err)
break out
}
tag := op >> 3
wire := op & 7
switch wire {
default:
fmt.Printf("%3d: t=%3d unknown wire=%d\n",
index, tag, wire)
break out
case WireBytes:
var r []byte
r, err = p.DecodeRawBytes(false)
if err != nil {
break out
}
fmt.Printf("%3d: t=%3d bytes [%d]", index, tag, len(r))
if len(r) <= 6 {
for i := 0; i < len(r); i++ {
fmt.Printf(" %.2x", r[i])
}
} else {
for i := 0; i < 3; i++ {
fmt.Printf(" %.2x", r[i])
}
fmt.Printf(" ..")
for i := len(r) - 3; i < len(r); i++ {
fmt.Printf(" %.2x", r[i])
}
}
fmt.Printf("\n")
case WireFixed32:
u, err = p.DecodeFixed32()
if err != nil {
fmt.Printf("%3d: t=%3d fix32 err %v\n", index, tag, err)
break out
}
fmt.Printf("%3d: t=%3d fix32 %d\n", index, tag, u)
case WireFixed64:
u, err = p.DecodeFixed64()
if err != nil {
fmt.Printf("%3d: t=%3d fix64 err %v\n", index, tag, err)
break out
}
fmt.Printf("%3d: t=%3d fix64 %d\n", index, tag, u)
case WireVarint:
u, err = p.DecodeVarint()
if err != nil {
fmt.Printf("%3d: t=%3d varint err %v\n", index, tag, err)
break out
}
fmt.Printf("%3d: t=%3d varint %d\n", index, tag, u)
case WireStartGroup:
fmt.Printf("%3d: t=%3d start\n", index, tag)
depth++
case WireEndGroup:
depth--
fmt.Printf("%3d: t=%3d end\n", index, tag)
}
}
if depth != 0 {
fmt.Printf("%3d: start-end not balanced %d\n", p.index, depth)
}
fmt.Printf("\n")
p.buf = obuf
p.index = index
}
// SetDefaults sets unset protocol buffer fields to their default values.
// It only modifies fields that are both unset and have defined defaults.
// It recursively sets default values in any non-nil sub-messages.
func SetDefaults(pb Message) {
setDefaults(reflect.ValueOf(pb), true, false)
}
// v is a pointer to a struct.
func setDefaults(v reflect.Value, recur, zeros bool) {
v = v.Elem()
defaultMu.RLock()
dm, ok := defaults[v.Type()]
defaultMu.RUnlock()
if !ok {
dm = buildDefaultMessage(v.Type())
defaultMu.Lock()
defaults[v.Type()] = dm
defaultMu.Unlock()
}
for _, sf := range dm.scalars {
f := v.Field(sf.index)
if !f.IsNil() {
// field already set
continue
}
dv := sf.value
if dv == nil && !zeros {
// no explicit default, and don't want to set zeros
continue
}
fptr := f.Addr().Interface() // **T
// TODO: Consider batching the allocations we do here.
switch sf.kind {
case reflect.Bool:
b := new(bool)
if dv != nil {
*b = dv.(bool)
}
*(fptr.(**bool)) = b
case reflect.Float32:
f := new(float32)
if dv != nil {
*f = dv.(float32)
}
*(fptr.(**float32)) = f
case reflect.Float64:
f := new(float64)
if dv != nil {
*f = dv.(float64)
}
*(fptr.(**float64)) = f
case reflect.Int32:
// might be an enum
if ft := f.Type(); ft != int32PtrType {
// enum
f.Set(reflect.New(ft.Elem()))
if dv != nil {
f.Elem().SetInt(int64(dv.(int32)))
}
} else {
// int32 field
i := new(int32)
if dv != nil {
*i = dv.(int32)
}
*(fptr.(**int32)) = i
}
case reflect.Int64:
i := new(int64)
if dv != nil {
*i = dv.(int64)
}
*(fptr.(**int64)) = i
case reflect.String:
s := new(string)
if dv != nil {
*s = dv.(string)
}
*(fptr.(**string)) = s
case reflect.Uint8:
// exceptional case: []byte
var b []byte
if dv != nil {
db := dv.([]byte)
b = make([]byte, len(db))
copy(b, db)
} else {
b = []byte{}
}
*(fptr.(*[]byte)) = b
case reflect.Uint32:
u := new(uint32)
if dv != nil {
*u = dv.(uint32)
}
*(fptr.(**uint32)) = u
case reflect.Uint64:
u := new(uint64)
if dv != nil {
*u = dv.(uint64)
}
*(fptr.(**uint64)) = u
default:
log.Printf("proto: can't set default for field %v (sf.kind=%v)", f, sf.kind)
}
}
for _, ni := range dm.nested {
f := v.Field(ni)
// f is *T or []*T or map[T]*T
switch f.Kind() {
case reflect.Ptr:
if f.IsNil() {
continue
}
setDefaults(f, recur, zeros)
case reflect.Slice:
for i := 0; i < f.Len(); i++ {
e := f.Index(i)
if e.IsNil() {
continue
}
setDefaults(e, recur, zeros)
}
case reflect.Map:
for _, k := range f.MapKeys() {
e := f.MapIndex(k)
if e.IsNil() {
continue
}
setDefaults(e, recur, zeros)
}
}
}
}
var (
// defaults maps a protocol buffer struct type to a slice of the fields,
// with its scalar fields set to their proto-declared non-zero default values.
defaultMu sync.RWMutex
defaults = make(map[reflect.Type]defaultMessage)
int32PtrType = reflect.TypeOf((*int32)(nil))
)
// defaultMessage represents information about the default values of a message.
type defaultMessage struct {
scalars []scalarField
nested []int // struct field index of nested messages
}
type scalarField struct {
index int // struct field index
kind reflect.Kind // element type (the T in *T or []T)
value interface{} // the proto-declared default value, or nil
}
// t is a struct type.
func buildDefaultMessage(t reflect.Type) (dm defaultMessage) {
sprop := GetProperties(t)
for _, prop := range sprop.Prop {
fi, ok := sprop.decoderTags.get(prop.Tag)
if !ok {
// XXX_unrecognized
continue
}
ft := t.Field(fi).Type
sf, nested, err := fieldDefault(ft, prop)
switch {
case err != nil:
log.Print(err)
case nested:
dm.nested = append(dm.nested, fi)
case sf != nil:
sf.index = fi
dm.scalars = append(dm.scalars, *sf)
}
}
return dm
}
// fieldDefault returns the scalarField for field type ft.
// sf will be nil if the field can not have a default.
// nestedMessage will be true if this is a nested message.
// Note that sf.index is not set on return.
func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMessage bool, err error) {
var canHaveDefault bool
switch ft.Kind() {
case reflect.Ptr:
if ft.Elem().Kind() == reflect.Struct {
nestedMessage = true
} else {
canHaveDefault = true // proto2 scalar field
}
case reflect.Slice:
switch ft.Elem().Kind() {
case reflect.Ptr:
nestedMessage = true // repeated message
case reflect.Uint8:
canHaveDefault = true // bytes field
}
case reflect.Map:
if ft.Elem().Kind() == reflect.Ptr {
nestedMessage = true // map with message values
}
}
if !canHaveDefault {
if nestedMessage {
return nil, true, nil
}
return nil, false, nil
}
// We now know that ft is a pointer or slice.
sf = &scalarField{kind: ft.Elem().Kind()}
// scalar fields without defaults
if !prop.HasDefault {
return sf, false, nil
}
// a scalar field: either *T or []byte
switch ft.Elem().Kind() {
case reflect.Bool:
x, err := strconv.ParseBool(prop.Default)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default bool %q: %v", prop.Default, err)
}
sf.value = x
case reflect.Float32:
x, err := strconv.ParseFloat(prop.Default, 32)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default float32 %q: %v", prop.Default, err)
}
sf.value = float32(x)
case reflect.Float64:
x, err := strconv.ParseFloat(prop.Default, 64)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default float64 %q: %v", prop.Default, err)
}
sf.value = x
case reflect.Int32:
x, err := strconv.ParseInt(prop.Default, 10, 32)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default int32 %q: %v", prop.Default, err)
}
sf.value = int32(x)
case reflect.Int64:
x, err := strconv.ParseInt(prop.Default, 10, 64)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default int64 %q: %v", prop.Default, err)
}
sf.value = x
case reflect.String:
sf.value = prop.Default
case reflect.Uint8:
// []byte (not *uint8)
sf.value = []byte(prop.Default)
case reflect.Uint32:
x, err := strconv.ParseUint(prop.Default, 10, 32)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default uint32 %q: %v", prop.Default, err)
}
sf.value = uint32(x)
case reflect.Uint64:
x, err := strconv.ParseUint(prop.Default, 10, 64)
if err != nil {
return nil, false, fmt.Errorf("proto: bad default uint64 %q: %v", prop.Default, err)
}
sf.value = x
default:
return nil, false, fmt.Errorf("proto: unhandled def kind %v", ft.Elem().Kind())
}
return sf, false, nil
}
// Map fields may have key types of non-float scalars, strings and enums.
// The easiest way to sort them in some deterministic order is to use fmt.
// If this turns out to be inefficient we can always consider other options,
// such as doing a Schwartzian transform.
func mapKeys(vs []reflect.Value) sort.Interface {
s := mapKeySorter{
vs: vs,
// default Less function: textual comparison
less: func(a, b reflect.Value) bool {
return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface())
},
}
// Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps;
// numeric keys are sorted numerically.
if len(vs) == 0 {
return s
}
switch vs[0].Kind() {
case reflect.Int32, reflect.Int64:
s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() }
case reflect.Uint32, reflect.Uint64:
s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() }
}
return s
}
type mapKeySorter struct {
vs []reflect.Value
less func(a, b reflect.Value) bool
}
func (s mapKeySorter) Len() int { return len(s.vs) }
func (s mapKeySorter) Swap(i, j int) { s.vs[i], s.vs[j] = s.vs[j], s.vs[i] }
func (s mapKeySorter) Less(i, j int) bool {
return s.less(s.vs[i], s.vs[j])
}
// isProto3Zero reports whether v is a zero proto3 value.
func isProto3Zero(v reflect.Value) bool {
switch v.Kind() {
case reflect.Bool:
return !v.Bool()
case reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint32, reflect.Uint64:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.String:
return v.String() == ""
}
return false
}
// ProtoPackageIsVersion1 is referenced from generated protocol buffer files
// to assert that that code is compatible with this version of the proto package.
const ProtoPackageIsVersion1 = true

@ -0,0 +1,280 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package proto
/*
* Support for message sets.
*/
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"reflect"
"sort"
)
// errNoMessageTypeID occurs when a protocol buffer does not have a message type ID.
// A message type ID is required for storing a protocol buffer in a message set.
var errNoMessageTypeID = errors.New("proto does not have a message type ID")
// The first two types (_MessageSet_Item and messageSet)
// model what the protocol compiler produces for the following protocol message:
// message MessageSet {
// repeated group Item = 1 {
// required int32 type_id = 2;
// required string message = 3;
// };
// }
// That is the MessageSet wire format. We can't use a proto to generate these
// because that would introduce a circular dependency between it and this package.
type _MessageSet_Item struct {
TypeId *int32 `protobuf:"varint,2,req,name=type_id"`
Message []byte `protobuf:"bytes,3,req,name=message"`
}
type messageSet struct {
Item []*_MessageSet_Item `protobuf:"group,1,rep"`
XXX_unrecognized []byte
// TODO: caching?
}
// Make sure messageSet is a Message.
var _ Message = (*messageSet)(nil)
// messageTypeIder is an interface satisfied by a protocol buffer type
// that may be stored in a MessageSet.
type messageTypeIder interface {
MessageTypeId() int32
}
func (ms *messageSet) find(pb Message) *_MessageSet_Item {
mti, ok := pb.(messageTypeIder)
if !ok {
return nil
}
id := mti.MessageTypeId()
for _, item := range ms.Item {
if *item.TypeId == id {
return item
}
}
return nil
}
func (ms *messageSet) Has(pb Message) bool {
if ms.find(pb) != nil {
return true
}
return false
}
func (ms *messageSet) Unmarshal(pb Message) error {
if item := ms.find(pb); item != nil {
return Unmarshal(item.Message, pb)
}
if _, ok := pb.(messageTypeIder); !ok {
return errNoMessageTypeID
}
return nil // TODO: return error instead?
}
func (ms *messageSet) Marshal(pb Message) error {
msg, err := Marshal(pb)
if err != nil {
return err
}
if item := ms.find(pb); item != nil {
// reuse existing item
item.Message = msg
return nil
}
mti, ok := pb.(messageTypeIder)
if !ok {
return errNoMessageTypeID
}
mtid := mti.MessageTypeId()
ms.Item = append(ms.Item, &_MessageSet_Item{
TypeId: &mtid,
Message: msg,
})
return nil
}
func (ms *messageSet) Reset() { *ms = messageSet{} }
func (ms *messageSet) String() string { return CompactTextString(ms) }
func (*messageSet) ProtoMessage() {}
// Support for the message_set_wire_format message option.
func skipVarint(buf []byte) []byte {
i := 0
for ; buf[i]&0x80 != 0; i++ {
}
return buf[i+1:]
}
// MarshalMessageSet encodes the extension map represented by m in the message set wire format.
// It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option.
func MarshalMessageSet(m map[int32]Extension) ([]byte, error) {
if err := encodeExtensionMap(m); err != nil {
return nil, err
}
// Sort extension IDs to provide a deterministic encoding.
// See also enc_map in encode.go.
ids := make([]int, 0, len(m))
for id := range m {
ids = append(ids, int(id))
}
sort.Ints(ids)
ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))}
for _, id := range ids {
e := m[int32(id)]
// Remove the wire type and field number varint, as well as the length varint.
msg := skipVarint(skipVarint(e.enc))
ms.Item = append(ms.Item, &_MessageSet_Item{
TypeId: Int32(int32(id)),
Message: msg,
})
}
return Marshal(ms)
}
// UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format.
// It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option.
func UnmarshalMessageSet(buf []byte, m map[int32]Extension) error {
ms := new(messageSet)
if err := Unmarshal(buf, ms); err != nil {
return err
}
for _, item := range ms.Item {
id := *item.TypeId
msg := item.Message
// Restore wire type and field number varint, plus length varint.
// Be careful to preserve duplicate items.
b := EncodeVarint(uint64(id)<<3 | WireBytes)
if ext, ok := m[id]; ok {
// Existing data; rip off the tag and length varint
// so we join the new data correctly.
// We can assume that ext.enc is set because we are unmarshaling.
o := ext.enc[len(b):] // skip wire type and field number
_, n := DecodeVarint(o) // calculate length of length varint
o = o[n:] // skip length varint
msg = append(o, msg...) // join old data and new data
}
b = append(b, EncodeVarint(uint64(len(msg)))...)
b = append(b, msg...)
m[id] = Extension{enc: b}
}
return nil
}
// MarshalMessageSetJSON encodes the extension map represented by m in JSON format.
// It is called by generated MarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) {
var b bytes.Buffer
b.WriteByte('{')
// Process the map in key order for deterministic output.
ids := make([]int32, 0, len(m))
for id := range m {
ids = append(ids, id)
}
sort.Sort(int32Slice(ids)) // int32Slice defined in text.go
for i, id := range ids {
ext := m[id]
if i > 0 {
b.WriteByte(',')
}
msd, ok := messageSetMap[id]
if !ok {
// Unknown type; we can't render it, so skip it.
continue
}
fmt.Fprintf(&b, `"[%s]":`, msd.name)
x := ext.value
if x == nil {
x = reflect.New(msd.t.Elem()).Interface()
if err := Unmarshal(ext.enc, x.(Message)); err != nil {
return nil, err
}
}
d, err := json.Marshal(x)
if err != nil {
return nil, err
}
b.Write(d)
}
b.WriteByte('}')
return b.Bytes(), nil
}
// UnmarshalMessageSetJSON decodes the extension map encoded in buf in JSON format.
// It is called by generated UnmarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
func UnmarshalMessageSetJSON(buf []byte, m map[int32]Extension) error {
// Common-case fast path.
if len(buf) == 0 || bytes.Equal(buf, []byte("{}")) {
return nil
}
// This is fairly tricky, and it's not clear that it is needed.
return errors.New("TODO: UnmarshalMessageSetJSON not yet implemented")
}
// A global registry of types that can be used in a MessageSet.
var messageSetMap = make(map[int32]messageSetDesc)
type messageSetDesc struct {
t reflect.Type // pointer to struct
name string
}
// RegisterMessageSetType is called from the generated code.
func RegisterMessageSetType(m Message, fieldNum int32, name string) {
messageSetMap[fieldNum] = messageSetDesc{
t: reflect.TypeOf(m),
name: name,
}
}

@ -0,0 +1,479 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2012 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// +build appengine
// This file contains an implementation of proto field accesses using package reflect.
// It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can
// be used on App Engine.
package proto
import (
"math"
"reflect"
)
// A structPointer is a pointer to a struct.
type structPointer struct {
v reflect.Value
}
// toStructPointer returns a structPointer equivalent to the given reflect value.
// The reflect value must itself be a pointer to a struct.
func toStructPointer(v reflect.Value) structPointer {
return structPointer{v}
}
// IsNil reports whether p is nil.
func structPointer_IsNil(p structPointer) bool {
return p.v.IsNil()
}
// Interface returns the struct pointer as an interface value.
func structPointer_Interface(p structPointer, _ reflect.Type) interface{} {
return p.v.Interface()
}
// A field identifies a field in a struct, accessible from a structPointer.
// In this implementation, a field is identified by the sequence of field indices
// passed to reflect's FieldByIndex.
type field []int
// toField returns a field equivalent to the given reflect field.
func toField(f *reflect.StructField) field {
return f.Index
}
// invalidField is an invalid field identifier.
var invalidField = field(nil)
// IsValid reports whether the field identifier is valid.
func (f field) IsValid() bool { return f != nil }
// field returns the given field in the struct as a reflect value.
func structPointer_field(p structPointer, f field) reflect.Value {
// Special case: an extension map entry with a value of type T
// passes a *T to the struct-handling code with a zero field,
// expecting that it will be treated as equivalent to *struct{ X T },
// which has the same memory layout. We have to handle that case
// specially, because reflect will panic if we call FieldByIndex on a
// non-struct.
if f == nil {
return p.v.Elem()
}
return p.v.Elem().FieldByIndex(f)
}
// ifield returns the given field in the struct as an interface value.
func structPointer_ifield(p structPointer, f field) interface{} {
return structPointer_field(p, f).Addr().Interface()
}
// Bytes returns the address of a []byte field in the struct.
func structPointer_Bytes(p structPointer, f field) *[]byte {
return structPointer_ifield(p, f).(*[]byte)
}
// BytesSlice returns the address of a [][]byte field in the struct.
func structPointer_BytesSlice(p structPointer, f field) *[][]byte {
return structPointer_ifield(p, f).(*[][]byte)
}
// Bool returns the address of a *bool field in the struct.
func structPointer_Bool(p structPointer, f field) **bool {
return structPointer_ifield(p, f).(**bool)
}
// BoolVal returns the address of a bool field in the struct.
func structPointer_BoolVal(p structPointer, f field) *bool {
return structPointer_ifield(p, f).(*bool)
}
// BoolSlice returns the address of a []bool field in the struct.
func structPointer_BoolSlice(p structPointer, f field) *[]bool {
return structPointer_ifield(p, f).(*[]bool)
}
// String returns the address of a *string field in the struct.
func structPointer_String(p structPointer, f field) **string {
return structPointer_ifield(p, f).(**string)
}
// StringVal returns the address of a string field in the struct.
func structPointer_StringVal(p structPointer, f field) *string {
return structPointer_ifield(p, f).(*string)
}
// StringSlice returns the address of a []string field in the struct.
func structPointer_StringSlice(p structPointer, f field) *[]string {
return structPointer_ifield(p, f).(*[]string)
}
// ExtMap returns the address of an extension map field in the struct.
func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension {
return structPointer_ifield(p, f).(*map[int32]Extension)
}
// NewAt returns the reflect.Value for a pointer to a field in the struct.
func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value {
return structPointer_field(p, f).Addr()
}
// SetStructPointer writes a *struct field in the struct.
func structPointer_SetStructPointer(p structPointer, f field, q structPointer) {
structPointer_field(p, f).Set(q.v)
}
// GetStructPointer reads a *struct field in the struct.
func structPointer_GetStructPointer(p structPointer, f field) structPointer {
return structPointer{structPointer_field(p, f)}
}
// StructPointerSlice the address of a []*struct field in the struct.
func structPointer_StructPointerSlice(p structPointer, f field) structPointerSlice {
return structPointerSlice{structPointer_field(p, f)}
}
// A structPointerSlice represents the address of a slice of pointers to structs
// (themselves messages or groups). That is, v.Type() is *[]*struct{...}.
type structPointerSlice struct {
v reflect.Value
}
func (p structPointerSlice) Len() int { return p.v.Len() }
func (p structPointerSlice) Index(i int) structPointer { return structPointer{p.v.Index(i)} }
func (p structPointerSlice) Append(q structPointer) {
p.v.Set(reflect.Append(p.v, q.v))
}
var (
int32Type = reflect.TypeOf(int32(0))
uint32Type = reflect.TypeOf(uint32(0))
float32Type = reflect.TypeOf(float32(0))
int64Type = reflect.TypeOf(int64(0))
uint64Type = reflect.TypeOf(uint64(0))
float64Type = reflect.TypeOf(float64(0))
)
// A word32 represents a field of type *int32, *uint32, *float32, or *enum.
// That is, v.Type() is *int32, *uint32, *float32, or *enum and v is assignable.
type word32 struct {
v reflect.Value
}
// IsNil reports whether p is nil.
func word32_IsNil(p word32) bool {
return p.v.IsNil()
}
// Set sets p to point at a newly allocated word with bits set to x.
func word32_Set(p word32, o *Buffer, x uint32) {
t := p.v.Type().Elem()
switch t {
case int32Type:
if len(o.int32s) == 0 {
o.int32s = make([]int32, uint32PoolSize)
}
o.int32s[0] = int32(x)
p.v.Set(reflect.ValueOf(&o.int32s[0]))
o.int32s = o.int32s[1:]
return
case uint32Type:
if len(o.uint32s) == 0 {
o.uint32s = make([]uint32, uint32PoolSize)
}
o.uint32s[0] = x
p.v.Set(reflect.ValueOf(&o.uint32s[0]))
o.uint32s = o.uint32s[1:]
return
case float32Type:
if len(o.float32s) == 0 {
o.float32s = make([]float32, uint32PoolSize)
}
o.float32s[0] = math.Float32frombits(x)
p.v.Set(reflect.ValueOf(&o.float32s[0]))
o.float32s = o.float32s[1:]
return
}
// must be enum
p.v.Set(reflect.New(t))
p.v.Elem().SetInt(int64(int32(x)))
}
// Get gets the bits pointed at by p, as a uint32.
func word32_Get(p word32) uint32 {
elem := p.v.Elem()
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
}
panic("unreachable")
}
// Word32 returns a reference to a *int32, *uint32, *float32, or *enum field in the struct.
func structPointer_Word32(p structPointer, f field) word32 {
return word32{structPointer_field(p, f)}
}
// A word32Val represents a field of type int32, uint32, float32, or enum.
// That is, v.Type() is int32, uint32, float32, or enum and v is assignable.
type word32Val struct {
v reflect.Value
}
// Set sets *p to x.
func word32Val_Set(p word32Val, x uint32) {
switch p.v.Type() {
case int32Type:
p.v.SetInt(int64(x))
return
case uint32Type:
p.v.SetUint(uint64(x))
return
case float32Type:
p.v.SetFloat(float64(math.Float32frombits(x)))
return
}
// must be enum
p.v.SetInt(int64(int32(x)))
}
// Get gets the bits pointed at by p, as a uint32.
func word32Val_Get(p word32Val) uint32 {
elem := p.v
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
}
panic("unreachable")
}
// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct.
func structPointer_Word32Val(p structPointer, f field) word32Val {
return word32Val{structPointer_field(p, f)}
}
// A word32Slice is a slice of 32-bit values.
// That is, v.Type() is []int32, []uint32, []float32, or []enum.
type word32Slice struct {
v reflect.Value
}
func (p word32Slice) Append(x uint32) {
n, m := p.v.Len(), p.v.Cap()
if n < m {
p.v.SetLen(n + 1)
} else {
t := p.v.Type().Elem()
p.v.Set(reflect.Append(p.v, reflect.Zero(t)))
}
elem := p.v.Index(n)
switch elem.Kind() {
case reflect.Int32:
elem.SetInt(int64(int32(x)))
case reflect.Uint32:
elem.SetUint(uint64(x))
case reflect.Float32:
elem.SetFloat(float64(math.Float32frombits(x)))
}
}
func (p word32Slice) Len() int {
return p.v.Len()
}
func (p word32Slice) Index(i int) uint32 {
elem := p.v.Index(i)
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
}
panic("unreachable")
}
// Word32Slice returns a reference to a []int32, []uint32, []float32, or []enum field in the struct.
func structPointer_Word32Slice(p structPointer, f field) word32Slice {
return word32Slice{structPointer_field(p, f)}
}
// word64 is like word32 but for 64-bit values.
type word64 struct {
v reflect.Value
}
func word64_Set(p word64, o *Buffer, x uint64) {
t := p.v.Type().Elem()
switch t {
case int64Type:
if len(o.int64s) == 0 {
o.int64s = make([]int64, uint64PoolSize)
}
o.int64s[0] = int64(x)
p.v.Set(reflect.ValueOf(&o.int64s[0]))
o.int64s = o.int64s[1:]
return
case uint64Type:
if len(o.uint64s) == 0 {
o.uint64s = make([]uint64, uint64PoolSize)
}
o.uint64s[0] = x
p.v.Set(reflect.ValueOf(&o.uint64s[0]))
o.uint64s = o.uint64s[1:]
return
case float64Type:
if len(o.float64s) == 0 {
o.float64s = make([]float64, uint64PoolSize)
}
o.float64s[0] = math.Float64frombits(x)
p.v.Set(reflect.ValueOf(&o.float64s[0]))
o.float64s = o.float64s[1:]
return
}
panic("unreachable")
}
func word64_IsNil(p word64) bool {
return p.v.IsNil()
}
func word64_Get(p word64) uint64 {
elem := p.v.Elem()
switch elem.Kind() {
case reflect.Int64:
return uint64(elem.Int())
case reflect.Uint64:
return elem.Uint()
case reflect.Float64:
return math.Float64bits(elem.Float())
}
panic("unreachable")
}
func structPointer_Word64(p structPointer, f field) word64 {
return word64{structPointer_field(p, f)}
}
// word64Val is like word32Val but for 64-bit values.
type word64Val struct {
v reflect.Value
}
func word64Val_Set(p word64Val, o *Buffer, x uint64) {
switch p.v.Type() {
case int64Type:
p.v.SetInt(int64(x))
return
case uint64Type:
p.v.SetUint(x)
return
case float64Type:
p.v.SetFloat(math.Float64frombits(x))
return
}
panic("unreachable")
}
func word64Val_Get(p word64Val) uint64 {
elem := p.v
switch elem.Kind() {
case reflect.Int64:
return uint64(elem.Int())
case reflect.Uint64:
return elem.Uint()
case reflect.Float64:
return math.Float64bits(elem.Float())
}
panic("unreachable")
}
func structPointer_Word64Val(p structPointer, f field) word64Val {
return word64Val{structPointer_field(p, f)}
}
type word64Slice struct {
v reflect.Value
}
func (p word64Slice) Append(x uint64) {
n, m := p.v.Len(), p.v.Cap()
if n < m {
p.v.SetLen(n + 1)
} else {
t := p.v.Type().Elem()
p.v.Set(reflect.Append(p.v, reflect.Zero(t)))
}
elem := p.v.Index(n)
switch elem.Kind() {
case reflect.Int64:
elem.SetInt(int64(int64(x)))
case reflect.Uint64:
elem.SetUint(uint64(x))
case reflect.Float64:
elem.SetFloat(float64(math.Float64frombits(x)))
}
}
func (p word64Slice) Len() int {
return p.v.Len()
}
func (p word64Slice) Index(i int) uint64 {
elem := p.v.Index(i)
switch elem.Kind() {
case reflect.Int64:
return uint64(elem.Int())
case reflect.Uint64:
return uint64(elem.Uint())
case reflect.Float64:
return math.Float64bits(float64(elem.Float()))
}
panic("unreachable")
}
func structPointer_Word64Slice(p structPointer, f field) word64Slice {
return word64Slice{structPointer_field(p, f)}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save