From c0ce859c76bcf2f059557e78bc69caaf6dc0c303 Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Thu, 29 Nov 2012 20:55:30 +0100 Subject: [PATCH] A few re-organizations. --- model/conversion.go | 95 ++++++++++++++++++++ model/metric.go | 15 ---- storage/metric/leveldb/leveldb.go | 140 ++++++++---------------------- utility/set.go | 4 +- 4 files changed, 134 insertions(+), 120 deletions(-) create mode 100644 model/conversion.go diff --git a/model/conversion.go b/model/conversion.go new file mode 100644 index 0000000000..63580c454f --- /dev/null +++ b/model/conversion.go @@ -0,0 +1,95 @@ +// Copyright 2012 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "code.google.com/p/goprotobuf/proto" + "crypto/md5" + "encoding/hex" + data "github.com/matttproud/prometheus/model/generated" + "io" + "sort" +) + +func SampleToMetricDDO(s *Sample) *data.MetricDDO { + labelLength := len(s.Labels) + labelNames := make([]string, 0, labelLength) + + for labelName := range s.Labels { + labelNames = append(labelNames, string(labelName)) + } + + sort.Strings(labelNames) + + labelPairs := make([]*data.LabelPairDDO, 0, labelLength) + + for _, labelName := range labelNames { + labelValue := s.Labels[labelName] + labelPair := &data.LabelPairDDO{ + Name: proto.String(string(labelName)), + Value: proto.String(string(labelValue)), + } + + labelPairs = append(labelPairs, labelPair) + } + + return &data.MetricDDO{ + LabelPair: labelPairs, + } +} + +func MetricToMetricDDO(m *Metric) *data.MetricDDO { + metricLength := len(*m) + labelNames := make([]string, 0, metricLength) + + for labelName := range *m { + labelNames = append(labelNames, string(labelName)) + } + + sort.Strings(labelNames) + + labelPairs := make([]*data.LabelPairDDO, 0, metricLength) + + for _, labelName := range labelNames { + labelValue := (*m)[labelName] + labelPair := &data.LabelPairDDO{ + Name: proto.String(string(labelName)), + Value: proto.String(string(labelValue)), + } + + labelPairs = append(labelPairs, labelPair) + } + + return &data.MetricDDO{ + LabelPair: labelPairs, + } +} + +func BytesToFingerprintDDO(b []byte) *data.FingerprintDDO { + return &data.FingerprintDDO{ + Signature: proto.String(string(b)), + } +} + +func StringToFingerprint(v string) Fingerprint { + hash := md5.New() + io.WriteString(hash, v) + return Fingerprint(hex.EncodeToString(hash.Sum([]byte{}))) +} + +func BytesToFingerprint(v []byte) Fingerprint { + hash := md5.New() + hash.Write(v) + return Fingerprint(hex.EncodeToString(hash.Sum([]byte{}))) +} diff --git a/model/metric.go b/model/metric.go index 25b5ef74d5..4b0e5e6458 100644 --- a/model/metric.go +++ b/model/metric.go @@ -14,9 +14,6 @@ package model import ( - "crypto/md5" - "encoding/hex" - "io" "time" ) @@ -42,15 +39,3 @@ type Interval struct { OldestInclusive time.Time NewestInclusive time.Time } - -func FingerprintFromString(value string) Fingerprint { - hash := md5.New() - io.WriteString(hash, value) - return Fingerprint(hex.EncodeToString(hash.Sum([]byte{}))) -} - -func FingerprintFromByteArray(value []byte) Fingerprint { - hash := md5.New() - hash.Write(value) - return Fingerprint(hex.EncodeToString(hash.Sum([]byte{}))) -} diff --git a/storage/metric/leveldb/leveldb.go b/storage/metric/leveldb/leveldb.go index d0f5080e75..2c5d8b4ec7 100644 --- a/storage/metric/leveldb/leveldb.go +++ b/storage/metric/leveldb/leveldb.go @@ -25,7 +25,6 @@ import ( "github.com/matttproud/prometheus/utility" "io" "log" - "sort" ) type LevelDBMetricPersistence struct { @@ -83,18 +82,20 @@ func (l *LevelDBMetricPersistence) Close() error { name := persistence.name closer := persistence.closer - if closer != nil { - log.Printf("Closing LevelDBPersistence storage container: %s\n", name) - closingError := closer.Close() + go func(name string, closer io.Closer) { + if closer != nil { + log.Printf("Closing LevelDBPersistence storage container: %s\n", name) + closingError := closer.Close() - if closingError != nil { - log.Printf("Could not close a LevelDBPersistence storage container; inconsistencies are possible: %q\n", closingError) + if closingError != nil { + log.Printf("Could not close a LevelDBPersistence storage container; inconsistencies are possible: %q\n", closingError) + } + + errorChannel <- closingError + } else { + errorChannel <- nil } - - errorChannel <- closingError - } else { - errorChannel <- nil - } + }(name, closer) } for i := 0; i < cap(errorChannel); i++ { @@ -124,57 +125,57 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc { "High-Water Marks by Fingerprint", func() { - var anomaly error - emission.fingerprintHighWaterMarks, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/high_water_marks_by_fingerprint", 1000000, 10) - errorChannel <- anomaly + var err error + emission.fingerprintHighWaterMarks, err = storage.NewLevelDBPersistence(baseDirectory+"/high_water_marks_by_fingerprint", 1000000, 10) + errorChannel <- err }, }, { "Label Names and Value Pairs by Fingerprint", func() { - var anomaly error - emission.fingerprintLabelPairs, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/label_name_and_value_pairs_by_fingerprint", 1000000, 10) - errorChannel <- anomaly + var err error + emission.fingerprintLabelPairs, err = storage.NewLevelDBPersistence(baseDirectory+"/label_name_and_value_pairs_by_fingerprint", 1000000, 10) + errorChannel <- err }, }, { "Low-Water Marks by Fingerprint", func() { - var anomaly error - emission.fingerprintLowWaterMarks, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/low_water_marks_by_fingerprint", 1000000, 10) - errorChannel <- anomaly + var err error + emission.fingerprintLowWaterMarks, err = storage.NewLevelDBPersistence(baseDirectory+"/low_water_marks_by_fingerprint", 1000000, 10) + errorChannel <- err }, }, { "Samples by Fingerprint", func() { - var anomaly error - emission.fingerprintSamples, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/samples_by_fingerprint", 1000000, 10) - errorChannel <- anomaly + var err error + emission.fingerprintSamples, err = storage.NewLevelDBPersistence(baseDirectory+"/samples_by_fingerprint", 1000000, 10) + errorChannel <- err }, }, { "Fingerprints by Label Name", func() { - var anomaly error - emission.labelNameFingerprints, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/fingerprints_by_label_name", 1000000, 10) - errorChannel <- anomaly + var err error + emission.labelNameFingerprints, err = storage.NewLevelDBPersistence(baseDirectory+"/fingerprints_by_label_name", 1000000, 10) + errorChannel <- err }, }, { "Fingerprints by Label Name and Value Pair", func() { - var anomaly error - emission.labelPairFingerprints, anomaly = storage.NewLevelDBPersistence(baseDirectory+"/fingerprints_by_label_name_and_value_pair", 1000000, 10) - errorChannel <- anomaly + var err error + emission.labelPairFingerprints, err = storage.NewLevelDBPersistence(baseDirectory+"/fingerprints_by_label_name_and_value_pair", 1000000, 10) + errorChannel <- err }, }, { "Metric Membership Index", func() { - var anomaly error - emission.metricMembershipIndex, anomaly = index.NewLevelDBMembershipIndex(baseDirectory+"/metric_membership_index", 1000000, 10) - errorChannel <- anomaly + var err error + emission.metricMembershipIndex, err = index.NewLevelDBMembershipIndex(baseDirectory+"/metric_membership_index", 1000000, 10) + errorChannel <- err }, }, } @@ -192,7 +193,6 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc openingError := <-errorChannel if openingError != nil { - log.Printf("Could not open a LevelDBPersistence storage container: %q\n", openingError) return nil, openingError @@ -204,70 +204,6 @@ func NewLevelDBMetricPersistence(baseDirectory string) (*LevelDBMetricPersistenc return emission, nil } -func ddoFromSample(sample *model.Sample) *data.MetricDDO { - labelNames := make([]string, 0, len(sample.Labels)) - - for labelName, _ := range sample.Labels { - labelNames = append(labelNames, string(labelName)) - } - - sort.Strings(labelNames) - - labelPairs := make([]*data.LabelPairDDO, 0, len(sample.Labels)) - - for _, labelName := range labelNames { - labelValue := sample.Labels[labelName] - labelPair := &data.LabelPairDDO{ - Name: proto.String(string(labelName)), - Value: proto.String(string(labelValue)), - } - - labelPairs = append(labelPairs, labelPair) - } - - metricDDO := &data.MetricDDO{ - LabelPair: labelPairs, - } - - return metricDDO -} - -func ddoFromMetric(metric model.Metric) *data.MetricDDO { - labelNames := make([]string, 0, len(metric)) - - for labelName, _ := range metric { - labelNames = append(labelNames, string(labelName)) - } - - sort.Strings(labelNames) - - labelPairs := make([]*data.LabelPairDDO, 0, len(metric)) - - for _, labelName := range labelNames { - labelValue := metric[labelName] - labelPair := &data.LabelPairDDO{ - Name: proto.String(string(labelName)), - Value: proto.String(string(labelValue)), - } - - labelPairs = append(labelPairs, labelPair) - } - - metricDDO := &data.MetricDDO{ - LabelPair: labelPairs, - } - - return metricDDO -} - -func fingerprintDDOFromByteArray(fingerprint []byte) *data.FingerprintDDO { - fingerprintDDO := &data.FingerprintDDO{ - Signature: proto.String(string(fingerprint)), - } - - return fingerprintDDO -} - func (l *LevelDBMetricPersistence) hasIndexMetric(ddo *data.MetricDDO) (bool, error) { ddoKey := coding.NewProtocolBufferEncoder(ddo) return l.metricMembershipIndex.Has(ddoKey) @@ -280,7 +216,7 @@ func (l *LevelDBMetricPersistence) indexMetric(ddo *data.MetricDDO) error { func fingerprintDDOForMessage(message proto.Message) (*data.FingerprintDDO, error) { if messageByteArray, marshalError := proto.Marshal(message); marshalError == nil { - fingerprint := model.FingerprintFromByteArray(messageByteArray) + fingerprint := model.BytesToFingerprint(messageByteArray) return &data.FingerprintDDO{ Signature: proto.String(string(fingerprint)), }, nil @@ -446,7 +382,7 @@ func (l *LevelDBMetricPersistence) appendFingerprints(ddo *data.MetricDDO) error } func (l *LevelDBMetricPersistence) AppendSample(sample *model.Sample) error { - metricDDO := ddoFromSample(sample) + metricDDO := model.SampleToMetricDDO(sample) if indexHas, indexHasError := l.hasIndexMetric(metricDDO); indexHasError == nil { if !indexHas { @@ -584,7 +520,7 @@ func (l *LevelDBMetricPersistence) GetMetrics() ([]model.LabelPairs, error) { } func (l *LevelDBMetricPersistence) GetWatermarksForMetric(metric model.Metric) (*model.Interval, int, error) { - metricDDO := ddoFromMetric(metric) + metricDDO := model.MetricToMetricDDO(&metric) if fingerprintDDO, fingerprintDDOErr := fingerprintDDOForMessage(metricDDO); fingerprintDDOErr == nil { if iterator, closer, iteratorErr := l.fingerprintSamples.GetIterator(); iteratorErr == nil { @@ -654,10 +590,8 @@ func (l *LevelDBMetricPersistence) GetWatermarksForMetric(metric model.Metric) ( return nil, -1, errors.New("Unknown error occurred while querying metric watermarks.") } -// TODO(mtp): Holes in the data! - func (l *LevelDBMetricPersistence) GetSamplesForMetric(metric model.Metric, interval model.Interval) ([]model.Samples, error) { - metricDDO := ddoFromMetric(metric) + metricDDO := model.MetricToMetricDDO(&metric) if fingerprintDDO, fingerprintDDOErr := fingerprintDDOForMessage(metricDDO); fingerprintDDOErr == nil { if iterator, closer, iteratorErr := l.fingerprintSamples.GetIterator(); iteratorErr == nil { diff --git a/utility/set.go b/utility/set.go index 97a60773b0..f2a565a72c 100644 --- a/utility/set.go +++ b/utility/set.go @@ -26,7 +26,7 @@ func (s Set) Remove(v interface{}) { func (s Set) Elements() []interface{} { result := make([]interface{}, 0, len(s)) - for k, _ := range s { + for k := range s { result = append(result, k) } @@ -42,7 +42,7 @@ func (s Set) Has(v interface{}) bool { func (s Set) Intersection(o Set) Set { result := make(Set) - for k, _ := range s { + for k := range s { if o.Has(k) { result[k] = true }