Minor optimize on log file collection (#16)

* *: minor optimize of coding style

* logfiles: fix bugs when collecting log files with ansible

* logfiles: fix wrong checking params when called by ansible

* makefile: add script to build release tarball

Author: AstroProfundis

.gitignore

@@ -2,6 +2,9 @@ bin/
 data/
 
 .vscode/
+.build/
+
+*.log
 
 # Binaries for programs and plugins
 *.exe

Makefile

@@ -9,3 +9,6 @@ all: default
 
 collector:
 	$(MAKE) -C collector $(MAKECMDGOALS)
+
+package:
+	./package.sh 2>package.err.log

collector/Makefile

@@ -5,8 +5,12 @@ ifeq "$(GOPATH)" ""
     $(error Please set the environment variable GOPATH before running `make`)
 endif
 
-GO        := go
-GOBUILD   := GOPATH=$(GOPATH) $(GO) build
+ifeq "$(GOBIN)" ""
+    GO    := GOPATH=$(GOPATH) go
+else
+    GO    := GOPATH=$(GOPATH) $(GOBIN)
+endif
+GOBUILD   := $(GO) build
 
 COMMIT    := $(shell git rev-parse HEAD)
 BRANCH    := $(shell git rev-parse --abbrev-ref HEAD)
@@ -26,6 +30,15 @@ debug:
 		-i *.go
 
 release:
+	$(GOBUILD) -ldflags '-w $(LDFLAGS)' \
+		-o ../bin/collector \
+		-i *.go
+
+deps:
+	$(GO) get ./...
+
+static:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
 	$(GOBUILD) -ldflags '-s -w -extldflags "-static" $(LDFLAGS)' \
 		-o ../bin/collector \
 		-i *.go

insight.py

@@ -48,8 +48,11 @@ class Insight():
     insight_trace = None
 
     def __init__(self, args):
-        if not args.alias:
+        if args.alias:
+            self.alias = args.alias
+        else:
             self.alias = util.get_hostname()
+
         if args.output and util.is_abs_path(args.output):
             self.outdir = args.output
             self.full_outdir = fileutils.create_dir(
@@ -195,12 +198,12 @@ def save_logfiles(self, args):
             return
         # reading logs requires root priviledge
         if not util.is_root_privilege():
-            logging.fatal("It's required to read logs with root priviledge.")
-            return
+            logging.warn("It's required to read logs with root priviledge.")
+            #return
 
         self.insight_logfiles = logfiles.InsightLogFiles(options=args)
-        proc_cmdline = self.format_proc_info("cmd")  # cmdline of process
         if args.log_auto:
+            proc_cmdline = self.format_proc_info("cmd")  # cmdline of process
             self.insight_logfiles.save_logfiles_auto(
                 proc_cmdline=proc_cmdline, outputdir=self.full_outdir)
         else:
@@ -245,9 +248,7 @@ def read_pdctl(self, args):
 
     insight = Insight(args)
 
-    if (not args.pid and not args.proc_listen_port
-        and not args.log_auto and not args.config_auto
-        ):
+    if (args.log_auto or args.config_auto):
         insight.collector()
         # check size of data folder of TiDB processes
         insight.get_datadir_size()
@@ -269,4 +270,4 @@ def read_pdctl(self, args):
 
     # compress all output to tarball
     if args.compress:
-        fileutils.compress_tarball(insight.full_outdir, insight.alias)
+        fileutils.compress_tarball(insight.outdir, insight.alias)

measurement/files/logfiles.py

@@ -123,7 +123,8 @@ def save_tidb_logfiles(self, outputdir=None):
         # the output tarball name
         output_name = "%s_%s" % (file_prefix, self.log_options.alias)
         # the full path of output directory
-        output_dir = os.path.join(output_base, output_name)
+        output_dir = fileutils.create_dir(
+            os.path.join(output_base, output_name))
 
         # copy valid log files to output directory
         file_list = self.get_filelist_in_time(source_dir, file_prefix,

measurement/perf.py

@@ -85,7 +85,7 @@ def run(self, outputdir=None):
         full_outputdir = fileutils.build_full_output_dir(
             basedir=outputdir, subdir=self.data_dir)
 
-        if full_outputdir is None:
+        if not full_outputdir:
             # something went wrong when setting output dir, exit without perfing
             # TODO: unified output: "Error when setting up output dir of perf data"
             return

package.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# make a release tarball
+
+if [ -z $1 ]; then
+  RELVER=`git describe --tags`
+else
+  RELVER=$1
+fi
+RELPATH=tidb-insight-${RELVER}
+
+GO_RELEASE_BIN=go1.10.3.linux-amd64
+
+BUILD_ROOT="`pwd`/.build"
+mkdir -p ${BUILD_ROOT}
+cd ${BUILD_ROOT}
+
+if [ ! -f ${GO_RELEASE_BIN}.tar.gz ]; then
+  wget https://dl.google.com/go/${GO_RELEASE_BIN}.tar.gz
+  tar zxf ${GO_RELEASE_BIN}.tar.gz
+fi
+
+GOROOT="${BUILD_ROOT}/go"
+GOPATH="${BUILD_ROOT}/.go"
+export GOROOT GOPATH
+
+# clean exist binaries
+rm -rf ${BUILD_ROOT}/tidb-insight-*
+mkdir -p ${BUILD_ROOT}/${RELPATH}/
+cp -rf ${BUILD_ROOT}/../* ${BUILD_ROOT}/${RELPATH}/
+
+cd ${BUILD_ROOT}/${RELPATH}/collector/
+
+# prepare dependencies
+GOBIN=${GOROOT}/bin/go make deps
+# compile a static binary
+GOBIN=${GOROOT}/bin/go make static
+
+# clean unecessary files
+cd ${BUILD_ROOT}/${RELPATH}
+rm -rf collector docs tests Makefile package.sh *.log
+find ${BUILD_ROOT}/${RELPATH} -name "*.pyc" | xargs rm
+
+# make tarball archive
+cd ${BUILD_ROOT}
+tar zcf ${RELPATH}.tar.gz ${RELPATH}