Compare commits
2 Commits
c8471ec264
...
ec44d7f643
Author | SHA1 | Date | |
---|---|---|---|
ec44d7f643 | |||
df80db0e29 |
@ -1,5 +1,7 @@
|
|||||||
# kubeprobes
|
# kubeprobes
|
||||||
|
|
||||||
|
[![Go Report Card](https://goreportcard.com/badge/pkg.icikowski.pl/kubeprobes)](https://goreportcard.com/report/pkg.icikowski.pl/kubeprobes)
|
||||||
|
|
||||||
Simple and effective package for implementing [Kubernetes liveness and readiness probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/)' handler.
|
Simple and effective package for implementing [Kubernetes liveness and readiness probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/)' handler.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
294
kubeprobes.go
294
kubeprobes.go
@ -1,147 +1,147 @@
|
|||||||
package kubeprobes
|
package kubeprobes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Kubeprobes represents liveness & readiness probes handler.
|
// Kubeprobes represents liveness & readiness probes handler.
|
||||||
type Kubeprobes interface {
|
type Kubeprobes interface {
|
||||||
http.Handler
|
http.Handler
|
||||||
|
|
||||||
// LivenessHandler returns [http.Handler] for liveness probes.
|
// LivenessHandler returns [http.Handler] for liveness probes.
|
||||||
LivenessHandler() http.Handler
|
LivenessHandler() http.Handler
|
||||||
// ReadinessHandler returns [http.Handler] for readiness probes.
|
// ReadinessHandler returns [http.Handler] for readiness probes.
|
||||||
ReadinessHandler() http.Handler
|
ReadinessHandler() http.Handler
|
||||||
}
|
}
|
||||||
|
|
||||||
type kubeprobes struct {
|
type kubeprobes struct {
|
||||||
livenessProbes []Probe
|
livenessProbes []Probe
|
||||||
readinessProbes []Probe
|
readinessProbes []Probe
|
||||||
|
|
||||||
verbose bool
|
verbose bool
|
||||||
|
|
||||||
pathLive string
|
pathLive string
|
||||||
pathReady string
|
pathReady string
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a new instance of a Kubernetes probes with given options.
|
// New returns a new instance of a Kubernetes probes with given options.
|
||||||
func New(options ...Option) (Kubeprobes, error) {
|
func New(options ...Option) (Kubeprobes, error) {
|
||||||
kp := &kubeprobes{
|
kp := &kubeprobes{
|
||||||
livenessProbes: []Probe{},
|
livenessProbes: []Probe{},
|
||||||
readinessProbes: []Probe{},
|
readinessProbes: []Probe{},
|
||||||
pathLive: defaultLivenessPath,
|
pathLive: defaultLivenessPath,
|
||||||
pathReady: defaultReadinessPath,
|
pathReady: defaultReadinessPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, option := range options {
|
for _, option := range options {
|
||||||
option.apply(kp)
|
option.apply(kp)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := kp.validate(); err != nil {
|
if err := kp.validate(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return kp, nil
|
return kp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (kp *kubeprobes) validate() error {
|
func (kp *kubeprobes) validate() error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
if kp.pathLive == "" {
|
if kp.pathLive == "" {
|
||||||
err = errors.Join(err, fmt.Errorf("liveness probe path must not be empty"))
|
err = errors.Join(err, fmt.Errorf("liveness probe path must not be empty"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if kp.pathReady == "" {
|
if kp.pathReady == "" {
|
||||||
err = errors.Join(err, fmt.Errorf("readiness probe path must not be empty"))
|
err = errors.Join(err, fmt.Errorf("readiness probe path must not be empty"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.HasPrefix(kp.pathLive, "/") {
|
if !strings.HasPrefix(kp.pathLive, "/") {
|
||||||
err = errors.Join(err, fmt.Errorf("liveness probe path must start with slash (current: %q)", kp.pathLive))
|
err = errors.Join(err, fmt.Errorf("liveness probe path must start with slash (current: %q)", kp.pathLive))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.HasPrefix(kp.pathReady, "/") {
|
if !strings.HasPrefix(kp.pathReady, "/") {
|
||||||
err = errors.Join(err, fmt.Errorf("readiness probe path must start with slash (current: %q)", kp.pathReady))
|
err = errors.Join(err, fmt.Errorf("readiness probe path must start with slash (current: %q)", kp.pathReady))
|
||||||
}
|
}
|
||||||
|
|
||||||
if kp.pathLive == kp.pathReady {
|
if kp.pathLive == kp.pathReady {
|
||||||
err = errors.Join(err, fmt.Errorf("liveness and readiness probes have the same values (both %q)", kp.pathLive))
|
err = errors.Join(err, fmt.Errorf("liveness and readiness probes have the same values (both %q)", kp.pathLive))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(kp.livenessProbes) == 0 {
|
if len(kp.livenessProbes) == 0 {
|
||||||
err = errors.Join(err, fmt.Errorf("no liveness probes defined"))
|
err = errors.Join(err, fmt.Errorf("no liveness probes defined"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(kp.readinessProbes) == 0 {
|
if len(kp.readinessProbes) == 0 {
|
||||||
err = errors.Join(err, fmt.Errorf("no readiness probes defined"))
|
err = errors.Join(err, fmt.Errorf("no readiness probes defined"))
|
||||||
}
|
}
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
type probesResponse struct {
|
type probesResponse struct {
|
||||||
Passed []statusEntry `json:"passed,omitempty"`
|
Passed []statusEntry `json:"passed,omitempty"`
|
||||||
Failed []statusEntry `json:"failed,omitempty"`
|
Failed []statusEntry `json:"failed,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (kp *kubeprobes) handleLiveness(w http.ResponseWriter, r *http.Request) {
|
func (kp *kubeprobes) handleLiveness(w http.ResponseWriter, r *http.Request) {
|
||||||
sq := newStatusQuery(kp.livenessProbes)
|
sq := newStatusQuery(kp.livenessProbes)
|
||||||
output := probesResponse{}
|
output := probesResponse{}
|
||||||
|
|
||||||
sq.wait()
|
sq.wait()
|
||||||
output.Failed = sq.failed
|
output.Failed = sq.failed
|
||||||
if r.URL.Query().Has(verboseOutputFlag) || kp.verbose {
|
if r.URL.Query().Has(verboseOutputFlag) || kp.verbose {
|
||||||
output.Passed = sq.passed
|
output.Passed = sq.passed
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Add(headerContentType, contentTypeJSON)
|
w.Header().Add(headerContentType, contentTypeJSON)
|
||||||
if sq.ok {
|
if sq.ok {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
} else {
|
} else {
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
}
|
}
|
||||||
_ = json.NewEncoder(w).Encode(output)
|
_ = json.NewEncoder(w).Encode(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (kp *kubeprobes) handleReadiness(w http.ResponseWriter, r *http.Request) {
|
func (kp *kubeprobes) handleReadiness(w http.ResponseWriter, r *http.Request) {
|
||||||
sq := newStatusQuery(append(kp.livenessProbes, kp.readinessProbes...))
|
sq := newStatusQuery(append(kp.livenessProbes, kp.readinessProbes...))
|
||||||
output := probesResponse{}
|
output := probesResponse{}
|
||||||
|
|
||||||
sq.wait()
|
sq.wait()
|
||||||
output.Failed = sq.failed
|
output.Failed = sq.failed
|
||||||
if r.URL.Query().Has(verboseOutputFlag) || kp.verbose {
|
if r.URL.Query().Has(verboseOutputFlag) || kp.verbose {
|
||||||
output.Passed = sq.passed
|
output.Passed = sq.passed
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Add(headerContentType, contentTypeJSON)
|
w.Header().Add(headerContentType, contentTypeJSON)
|
||||||
if sq.ok {
|
if sq.ok {
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
} else {
|
} else {
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
}
|
}
|
||||||
_ = json.NewEncoder(w).Encode(output)
|
_ = json.NewEncoder(w).Encode(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
// LivenessHandler implements Kubeprobes.
|
// LivenessHandler implements Kubeprobes.
|
||||||
func (kp *kubeprobes) LivenessHandler() http.Handler {
|
func (kp *kubeprobes) LivenessHandler() http.Handler {
|
||||||
return http.HandlerFunc(kp.handleLiveness)
|
return http.HandlerFunc(kp.handleLiveness)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadinessHandler implements Kubeprobes.
|
// ReadinessHandler implements Kubeprobes.
|
||||||
func (kp *kubeprobes) ReadinessHandler() http.Handler {
|
func (kp *kubeprobes) ReadinessHandler() http.Handler {
|
||||||
return http.HandlerFunc(kp.handleReadiness)
|
return http.HandlerFunc(kp.handleReadiness)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP implements Kubeprobes.
|
// ServeHTTP implements Kubeprobes.
|
||||||
func (kp *kubeprobes) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (kp *kubeprobes) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
switch r.URL.Path {
|
switch r.URL.Path {
|
||||||
case kp.pathLive:
|
case kp.pathLive:
|
||||||
kp.handleLiveness(w, r)
|
kp.handleLiveness(w, r)
|
||||||
case kp.pathReady:
|
case kp.pathReady:
|
||||||
kp.handleReadiness(w, r)
|
kp.handleReadiness(w, r)
|
||||||
default:
|
default:
|
||||||
w.WriteHeader(http.StatusNotFound)
|
w.WriteHeader(http.StatusNotFound)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,182 +1,182 @@
|
|||||||
package kubeprobes
|
package kubeprobes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getStatusFromEndpoint(t *testing.T, client *http.Client, endpoint string) int {
|
func getStatusFromEndpoint(t *testing.T, client *http.Client, endpoint string) int {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
resp, err := client.Get(endpoint)
|
resp, err := client.Get(endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("error getting status from endpoint: %s", err)
|
t.Errorf("error getting status from endpoint: %s", err)
|
||||||
}
|
}
|
||||||
return resp.StatusCode
|
return resp.StatusCode
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidation(t *testing.T) {
|
func TestValidation(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
live, _ = NewManualProbe("live")
|
live, _ = NewManualProbe("live")
|
||||||
ready, _ = NewManualProbe("ready")
|
ready, _ = NewManualProbe("ready")
|
||||||
)
|
)
|
||||||
|
|
||||||
tests := map[string]struct {
|
tests := map[string]struct {
|
||||||
opts []Option
|
opts []Option
|
||||||
expectedError bool
|
expectedError bool
|
||||||
}{
|
}{
|
||||||
"no modifications and no error": {
|
"no modifications and no error": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"modifications and no error": {
|
"modifications and no error": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithLivenessPath("/livez"),
|
WithLivenessPath("/livez"),
|
||||||
WithReadinessPath("/readyz"),
|
WithReadinessPath("/readyz"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"missing liveness probes": {
|
"missing liveness probes": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"missing readiness probes": {
|
"missing readiness probes": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"liveness probe path empty": {
|
"liveness probe path empty": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithLivenessPath(""),
|
WithLivenessPath(""),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"readiness probe path empty": {
|
"readiness probe path empty": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithReadinessPath(""),
|
WithReadinessPath(""),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"liveness probe path does not start with slash": {
|
"liveness probe path does not start with slash": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithLivenessPath("livez"),
|
WithLivenessPath("livez"),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"readiness probe path does not start with slash": {
|
"readiness probe path does not start with slash": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithReadinessPath("readyz"),
|
WithReadinessPath("readyz"),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
"liveness and readiness probe paths are equal": {
|
"liveness and readiness probe paths are equal": {
|
||||||
opts: []Option{
|
opts: []Option{
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
WithLivenessPath("/check"),
|
WithLivenessPath("/check"),
|
||||||
WithReadinessPath("/check"),
|
WithReadinessPath("/check"),
|
||||||
},
|
},
|
||||||
expectedError: true,
|
expectedError: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, tc := range tests {
|
for name, tc := range tests {
|
||||||
name, tc := name, tc
|
name, tc := name, tc
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
_, err := New(tc.opts...)
|
_, err := New(tc.opts...)
|
||||||
switch {
|
switch {
|
||||||
case err == nil && tc.expectedError:
|
case err == nil && tc.expectedError:
|
||||||
t.Error("expected error, but no error was returned")
|
t.Error("expected error, but no error was returned")
|
||||||
case err != nil && !tc.expectedError:
|
case err != nil && !tc.expectedError:
|
||||||
t.Errorf("expected no error but got %v", err)
|
t.Errorf("expected no error but got %v", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandler(t *testing.T) {
|
func TestHandler(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
live, _ = NewManualProbe("live")
|
live, _ = NewManualProbe("live")
|
||||||
ready, _ = NewManualProbe("ready")
|
ready, _ = NewManualProbe("ready")
|
||||||
)
|
)
|
||||||
|
|
||||||
tests := map[string]struct {
|
tests := map[string]struct {
|
||||||
livenessProbeTransformation func(*testing.T, ManualProbe)
|
livenessProbeTransformation func(*testing.T, ManualProbe)
|
||||||
readinessProbeTransformation func(*testing.T, ManualProbe)
|
readinessProbeTransformation func(*testing.T, ManualProbe)
|
||||||
expectedLiveStatus int
|
expectedLiveStatus int
|
||||||
expectedReadyStatus int
|
expectedReadyStatus int
|
||||||
}{
|
}{
|
||||||
"not live": {
|
"not live": {
|
||||||
livenessProbeTransformation: markAsDown,
|
livenessProbeTransformation: markAsDown,
|
||||||
readinessProbeTransformation: markAsDown,
|
readinessProbeTransformation: markAsDown,
|
||||||
expectedLiveStatus: http.StatusServiceUnavailable,
|
expectedLiveStatus: http.StatusServiceUnavailable,
|
||||||
expectedReadyStatus: http.StatusServiceUnavailable,
|
expectedReadyStatus: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
"live but not ready": {
|
"live but not ready": {
|
||||||
livenessProbeTransformation: markAsUp,
|
livenessProbeTransformation: markAsUp,
|
||||||
readinessProbeTransformation: markAsDown,
|
readinessProbeTransformation: markAsDown,
|
||||||
expectedLiveStatus: http.StatusOK,
|
expectedLiveStatus: http.StatusOK,
|
||||||
expectedReadyStatus: http.StatusServiceUnavailable,
|
expectedReadyStatus: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
"live and ready": {
|
"live and ready": {
|
||||||
livenessProbeTransformation: markAsUp,
|
livenessProbeTransformation: markAsUp,
|
||||||
readinessProbeTransformation: markAsUp,
|
readinessProbeTransformation: markAsUp,
|
||||||
expectedLiveStatus: http.StatusOK,
|
expectedLiveStatus: http.StatusOK,
|
||||||
expectedReadyStatus: http.StatusOK,
|
expectedReadyStatus: http.StatusOK,
|
||||||
},
|
},
|
||||||
"ready but not live - should never happen": {
|
"ready but not live - should never happen": {
|
||||||
livenessProbeTransformation: markAsDown,
|
livenessProbeTransformation: markAsDown,
|
||||||
readinessProbeTransformation: markAsUp,
|
readinessProbeTransformation: markAsUp,
|
||||||
expectedLiveStatus: http.StatusServiceUnavailable,
|
expectedLiveStatus: http.StatusServiceUnavailable,
|
||||||
expectedReadyStatus: http.StatusServiceUnavailable,
|
expectedReadyStatus: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
kp, err := New(
|
kp, err := New(
|
||||||
WithLivenessProbes(live),
|
WithLivenessProbes(live),
|
||||||
WithReadinessProbes(ready),
|
WithReadinessProbes(ready),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("expected no error, got %v", err)
|
t.Errorf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
srv := httptest.NewServer(kp)
|
srv := httptest.NewServer(kp)
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
client := srv.Client()
|
client := srv.Client()
|
||||||
|
|
||||||
for name, test := range tests {
|
for name, test := range tests {
|
||||||
name, test := name, test
|
name, test := name, test
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
test.livenessProbeTransformation(t, live)
|
test.livenessProbeTransformation(t, live)
|
||||||
test.readinessProbeTransformation(t, ready)
|
test.readinessProbeTransformation(t, ready)
|
||||||
|
|
||||||
liveStatus := getStatusFromEndpoint(t, client, srv.URL+defaultLivenessPath)
|
liveStatus := getStatusFromEndpoint(t, client, srv.URL+defaultLivenessPath)
|
||||||
readyStatus := getStatusFromEndpoint(t, client, srv.URL+defaultReadinessPath)
|
readyStatus := getStatusFromEndpoint(t, client, srv.URL+defaultReadinessPath)
|
||||||
otherStatus := getStatusFromEndpoint(t, client, srv.URL+"/something")
|
otherStatus := getStatusFromEndpoint(t, client, srv.URL+"/something")
|
||||||
|
|
||||||
if liveStatus != test.expectedLiveStatus {
|
if liveStatus != test.expectedLiveStatus {
|
||||||
t.Errorf("expected live status %d, got %d", test.expectedLiveStatus, liveStatus)
|
t.Errorf("expected live status %d, got %d", test.expectedLiveStatus, liveStatus)
|
||||||
}
|
}
|
||||||
if readyStatus != test.expectedReadyStatus {
|
if readyStatus != test.expectedReadyStatus {
|
||||||
t.Errorf("expected ready status %d, got %d", test.expectedReadyStatus, readyStatus)
|
t.Errorf("expected ready status %d, got %d", test.expectedReadyStatus, readyStatus)
|
||||||
}
|
}
|
||||||
if otherStatus != http.StatusNotFound {
|
if otherStatus != http.StatusNotFound {
|
||||||
t.Errorf("expected 404 status, got %d", otherStatus)
|
t.Errorf("expected 404 status, got %d", otherStatus)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user