diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 00000000..0030d4f2 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,2 @@ +.gcloudignore +.git diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8339fd61..00000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Add no patterns to .hgignore except for files generated by the build. -last-change diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 15167cd7..00000000 --- a/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -# This source code refers to The Go Authors for copyright purposes. -# The master list of authors is in the main Go distribution, -# visible at http://tip.golang.org/AUTHORS. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 88dff59b..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,31 +0,0 @@ -# Contributing to Go - -Go is an open source project. - -It is the work of hundreds of contributors. We appreciate your help! - - -## Filing issues - -When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions: - -1. What version of Go are you using (`go version`)? -2. What operating system and processor architecture are you using? -3. What did you do? -4. What did you expect to see? -5. What did you see instead? - -General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker. -The gophers there will answer or ask you to file an issue if you've tripped over a bug. - -## Contributing code - -Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) -before sending patches. - -**We do not accept GitHub pull requests** -(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review). - -Unless otherwise noted, the Go source files are distributed under -the BSD-style license found in the LICENSE file. - diff --git a/CONTRIBUTORS b/CONTRIBUTORS deleted file mode 100644 index 1c4577e9..00000000 --- a/CONTRIBUTORS +++ /dev/null @@ -1,3 +0,0 @@ -# This source code was written by the Go contributors. -# The master list of contributors is in the main Go distribution, -# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 6a66aea5..00000000 --- a/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/PATENTS b/PATENTS deleted file mode 100644 index 73309904..00000000 --- a/PATENTS +++ /dev/null @@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the Go project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of Go, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of Go. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of Go or any code incorporated within this -implementation of Go constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of Go -shall terminate as of the date such litigation is filed. diff --git a/README b/README deleted file mode 100644 index 0ba461f4..00000000 --- a/README +++ /dev/null @@ -1,3 +0,0 @@ -This repository holds the Go Blog server code and content. - -To submit changes to this repository, see http://golang.org/doc/contribute.html. diff --git a/README.md b/README.md new file mode 100644 index 00000000..a1230562 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Go Blog (obsolete) + +This repository formerly held the Go Blog server code and content. + +That material has since moved to [golang.org/x/website](https://golang.org/x/website). diff --git a/app.yaml b/app.yaml deleted file mode 100644 index 882f5c9a..00000000 --- a/app.yaml +++ /dev/null @@ -1,13 +0,0 @@ -application: golang-org -version: blog-test -runtime: go -api_version: go1 - -handlers: -- url: /favicon.ico - static_files: static/favicon.ico - upload: static/favicon.ico -- url: /.* - script: _go_app - -nobuild_files: ^(support|content)/ diff --git a/blog/appengine.go b/blog/appengine.go deleted file mode 100644 index 12bcdf21..00000000 --- a/blog/appengine.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build appengine - -// This file implements an App Engine blog server. - -package main - -import ( - "net/http" - - "golang.org/x/tools/blog" -) - -func init() { - config.ContentPath = "content/" - config.TemplatePath = "template/" - s, err := blog.NewServer(config) - if err != nil { - panic(err) - } - http.Handle("/", s) -} diff --git a/blog/blog.go b/blog/blog.go deleted file mode 100644 index 08125df5..00000000 --- a/blog/blog.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Command blog is a web server for the Go blog that can run on App Engine or -// as a stand-alone HTTP server. -package main - -import ( - "net/http" - "strings" - "time" - - "golang.org/x/tools/blog" - "golang.org/x/tools/godoc/static" - - _ "golang.org/x/tools/playground" -) - -const hostname = "blog.golang.org" // default hostname for blog server - -var config = blog.Config{ - Hostname: hostname, - BaseURL: "//" + hostname, - GodocURL: "//golang.org", - HomeArticles: 5, // articles to display on the home page - FeedArticles: 10, // articles to include in Atom and JSON feeds - PlayEnabled: true, - FeedTitle: "The Go Programming Language Blog", -} - -func init() { - // Redirect "/blog/" to "/", because the menu bar link is to "/blog/" - // but we're serving from the root. - redirect := func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/", http.StatusFound) - } - http.HandleFunc("/blog", redirect) - http.HandleFunc("/blog/", redirect) - - http.Handle("/lib/godoc/", http.StripPrefix("/lib/godoc/", http.HandlerFunc(staticHandler))) -} - -func staticHandler(w http.ResponseWriter, r *http.Request) { - name := r.URL.Path - b, ok := static.Files[name] - if !ok { - http.NotFound(w, r) - return - } - http.ServeContent(w, r, name, time.Time{}, strings.NewReader(b)) -} diff --git a/blog/local.go b/blog/local.go deleted file mode 100644 index 2e7ab544..00000000 --- a/blog/local.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine - -// This file implements a stand-alone blog server. - -package main - -import ( - "flag" - "log" - "net/http" - - "golang.org/x/tools/blog" -) - -var ( - httpAddr = flag.String("http", "localhost:8080", "HTTP listen address") - contentPath = flag.String("content", "content/", "path to content files") - templatePath = flag.String("template", "template/", "path to template files") - staticPath = flag.String("static", "static/", "path to static files") - reload = flag.Bool("reload", false, "reload content on each page load") -) - -func main() { - flag.Parse() - config.ContentPath = *contentPath - config.TemplatePath = *templatePath - if *reload { - http.HandleFunc("/", reloadingBlogServer) - } else { - s, err := blog.NewServer(config) - if err != nil { - log.Fatal(err) - } - http.Handle("/", s) - } - fs := http.FileServer(http.Dir(*staticPath)) - http.Handle("/static/", http.StripPrefix("/static/", fs)) - log.Fatal(http.ListenAndServe(*httpAddr, nil)) -} - -// reloadingBlogServer is an handler that restarts the blog server on each page -// view. Inefficient; don't enable by default. Handy when editing blog content. -func reloadingBlogServer(w http.ResponseWriter, r *http.Request) { - s, err := blog.NewServer(config) - if err != nil { - http.Error(w, err.Error(), 500) - return - } - s.ServeHTTP(w, r) -} diff --git a/blog/rewrite.go b/blog/rewrite.go deleted file mode 100644 index 6ab91216..00000000 --- a/blog/rewrite.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "net/http" - -// Register HTTP handlers that redirect old blog paths to their new locations. -func init() { - for p := range urlMap { - dest := "/" + urlMap[p] - http.HandleFunc(p, func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, dest, http.StatusMovedPermanently) - }) - } -} - -var urlMap = map[string]string{ - "/2010/03/go-whats-new-in-march-2010.html": "go-whats-new-in-march-2010", - "/2010/04/json-rpc-tale-of-interfaces.html": "json-rpc-tale-of-interfaces", - "/2010/04/third-party-libraries-goprotobuf-and.html": "third-party-libraries-goprotobuf-and", - "/2010/05/go-at-io-frequently-asked-questions.html": "go-at-io-frequently-asked-questions", - "/2010/05/new-talk-and-tutorials.html": "new-talk-and-tutorials", - "/2010/05/upcoming-google-io-go-events.html": "upcoming-google-io-go-events", - "/2010/06/go-programming-session-video-from.html": "go-programming-session-video-from", - "/2010/07/gos-declaration-syntax.html": "gos-declaration-syntax", - "/2010/07/share-memory-by-communicating.html": "share-memory-by-communicating", - "/2010/08/defer-panic-and-recover.html": "defer-panic-and-recover", - "/2010/09/go-concurrency-patterns-timing-out-and.html": "go-concurrency-patterns-timing-out-and", - "/2010/09/go-wins-2010-bossie-award.html": "go-wins-2010-bossie-award", - "/2010/09/introducing-go-playground.html": "introducing-go-playground", - "/2010/10/real-go-projects-smarttwitter-and-webgo.html": "real-go-projects-smarttwitter-and-webgo", - "/2010/11/debugging-go-code-status-report.html": "debugging-go-code-status-report", - "/2010/11/go-one-year-ago-today.html": "go-one-year-ago-today", - "/2011/01/go-slices-usage-and-internals.html": "go-slices-usage-and-internals", - "/2011/01/json-and-go.html": "json-and-go", - "/2011/03/c-go-cgo.html": "c-go-cgo", - "/2011/03/go-becomes-more-stable.html": "go-becomes-more-stable", - "/2011/03/gobs-of-data.html": "gobs-of-data", - "/2011/03/godoc-documenting-go-code.html": "godoc-documenting-go-code", - "/2011/04/go-at-heroku.html": "go-at-heroku", - "/2011/04/introducing-gofix.html": "introducing-gofix", - "/2011/05/gif-decoder-exercise-in-go-interfaces.html": "gif-decoder-exercise-in-go-interfaces", - "/2011/05/go-and-google-app-engine.html": "go-and-google-app-engine", - "/2011/05/go-at-google-io-2011-videos.html": "go-at-google-io-2011-videos", - "/2011/06/first-class-functions-in-go-and-new-go.html": "first-class-functions-in-go-and-new-go", - "/2011/06/profiling-go-programs.html": "profiling-go-programs", - "/2011/06/spotlight-on-external-go-libraries.html": "spotlight-on-external-go-libraries", - "/2011/07/error-handling-and-go.html": "error-handling-and-go", - "/2011/07/go-for-app-engine-is-now-generally.html": "go-for-app-engine-is-now-generally", - "/2011/09/go-image-package.html": "go-image-package", - "/2011/09/go-imagedraw-package.html": "go-imagedraw-package", - "/2011/09/laws-of-reflection.html": "laws-of-reflection", - "/2011/09/two-go-talks-lexical-scanning-in-go-and.html": "two-go-talks-lexical-scanning-in-go-and", - "/2011/10/debugging-go-programs-with-gnu-debugger.html": "debugging-go-programs-with-gnu-debugger", - "/2011/10/go-app-engine-sdk-155-released.html": "go-app-engine-sdk-155-released", - "/2011/10/learn-go-from-your-browser.html": "learn-go-from-your-browser", - "/2011/10/preview-of-go-version-1.html": "preview-of-go-version-1", - "/2011/11/go-programming-language-turns-two.html": "go-programming-language-turns-two", - "/2011/11/writing-scalable-app-engine.html": "writing-scalable-app-engine", - "/2011/12/building-stathat-with-go.html": "building-stathat-with-go", - "/2011/12/from-zero-to-go-launching-on-google.html": "from-zero-to-go-launching-on-google", - "/2011/12/getting-to-know-go-community.html": "getting-to-know-go-community", - "/2012/03/go-version-1-is-released.html": "go-version-1-is-released", - "/2012/07/gccgo-in-gcc-471.html": "gccgo-in-gcc-471", - "/2012/07/go-videos-from-google-io-2012.html": "go-videos-from-google-io-2012", - "/2012/08/go-updates-in-app-engine-171.html": "go-updates-in-app-engine-171", - "/2012/08/organizing-go-code.html": "organizing-go-code", - "/2012/11/go-turns-three.html": "go-turns-three", - "/2013/01/concurrency-is-not-parallelism.html": "concurrency-is-not-parallelism", - "/2013/01/go-fmt-your-code.html": "go-fmt-your-code", - "/2013/01/the-app-engine-sdk-and-workspaces-gopath.html": "the-app-engine-sdk-and-workspaces-gopath", - "/2013/01/two-recent-go-talks.html": "two-recent-go-talks", - "/2013/02/getthee-to-go-meetup.html": "getthee-to-go-meetup", - "/2013/02/go-maps-in-action.html": "go-maps-in-action", - "/2013/03/two-recent-go-articles.html": "two-recent-go-articles", - "/2013/03/the-path-to-go-1.html": "the-path-to-go-1", - "/2013/05/go-11-is-released.html": "go-11-is-released.article", - "/2013/05/advanced-go-concurrency-patterns.html": "advanced-go-concurrency-patterns.article", -} diff --git a/codereview.cfg b/codereview.cfg deleted file mode 100644 index 3f8b14b6..00000000 --- a/codereview.cfg +++ /dev/null @@ -1 +0,0 @@ -issuerepo: golang/go diff --git a/content/4years-gopher.png b/content/4years-gopher.png deleted file mode 100644 index a8110150..00000000 Binary files a/content/4years-gopher.png and /dev/null differ diff --git a/content/4years-graph.png b/content/4years-graph.png deleted file mode 100644 index a5a26a14..00000000 Binary files a/content/4years-graph.png and /dev/null differ diff --git a/content/4years.article b/content/4years.article deleted file mode 100644 index 4c99dbb8..00000000 --- a/content/4years.article +++ /dev/null @@ -1,78 +0,0 @@ -Four years of Go -10 Nov 2013 -Tags: community, birthday - -Andrew Gerrand - -* Introduction - -Today marks the fourth anniversary of Go as an open source project. - -.image 4years-gopher.png - -Rather than talk about our technical progress (there'll be much to talk about -when we release Go 1.2 in a couple of weeks) we thought we would instead take -this occasion to look at how the Go community has grown. - -Let's start with a chart: - -.image 4years-graph.png - -This chart shows the growth of Google searches for the term -"[[http://www.google.com/trends/explore?hl=en-US#q=golang&date=10/2009+50m&cmpt=q][golang]]" -over the past four years. -Notice the knee in the curve around March 2012, when Go 1.0 was released. -If these searches are a decent proxy for interest, then it's clear that -interest in Go has grown remarkably since launch, and particularly so in the -last 2 years. - -But where is the interest coming from? - -The open source community has embraced Go, with our community wiki listing [[http://golang.org/wiki/Projects][hundreds of Go projects]]. Some popular ones: - -- [[http://docker.io][Docker]] is a tool for packaging and running applications in lightweight containers. Docker makes it easy to isolate, package, and deploy applications, and is beloved by system administrators. Its creator Solomon Hykes cited Go's standard library, concurrency primitives, and ease of deployment as key factors, and said "To put it simply, if Docker had not been written in Go, it would not have been as successful." - -- [[http://packer.io][Packer]] is a tool for automating the creation of machine images for deployment to virtual machines or cloud services. Its author, Mitchell Hashimoto, is now working on another Go project, [[http://www.serfdom.io/][serf]], a decentralized discovery service. Like Docker, these projects help with management of large-scale, cluster-based services. - -- [[http://bit.ly][Bitly]]'s [[http://bitly.github.io/nsq/][NSQ]] is a realtime distributed messaging platform designed for fault-tolerance and high-availability, and is used in production at bitly and a bunch of other companies. - -- [[http://canonical.com/][Canonical]]'s [[https://juju.ubuntu.com/][JuJu]] infrastructure automation system was rewritten in Go. Project lead Gustavo Niemeyer said "It's not a single aspect of Go that makes it a compelling choice, but rather the careful organization of well-crafted small pieces." - -- The [[https://github.com/goraft/raft][raft]] package provides an implementation of the [[https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf][Raft]] distributed consensus protocol. It is the basis of Go projects like [[https://github.com/coreos/etcd][etcd]] and [[https://github.com/skynetservices/skydns][SkyDNS]]. - -- Other popular projects include [[https://github.com/biogo/biogo][biogo]], the [[http://www.gorillatoolkit.org/][Gorilla Web Toolkit]], [[https://github.com/golang/groupcache][groupcache]], Mozilla's [[https://github.com/mozilla-services/heka][heka]], the [[https://github.com/cznic/kv][kv]] and [[https://github.com/cznic/ql][ql]] lightweight storage systems, and the [[http://skydb.io/][Sky]] behavioral database. - -But this is just the tip of the iceberg. The number of high-quality open source Go projects is phenomenal. Prolific Go hacker [[http://xph.us/software/][Keith Rarick]] put it well: "The state of the Go ecosystem after only four years is astounding. Compare Go in 2013 to Python in 1995 or Java in 1999. Or C++ in 1987!" - -Businesses are enjoying Go, too. The [[http://golang.org/wiki/GoUsers][Go Users wiki page]] lists dozens of success stories (and if you use Go, please add yourself to it). Some examples: - -- [[http://blog.cloudflare.com/go-at-cloudflare][CloudFlare]] built their distributed DNS service entirely with Go, and are in the process of migrating their gigabytes-per-minute logging infrastructure to the language. Programmer John Graham-Cumming said "We've found Go to be the perfect match for our needs: the combination of familiar syntax, a powerful type system, a strong network library and built-in concurrency means that more and more projects are being built here in Go." - -- [[http://soundcloud.com][SoundCloud]] is an audio distribution service that has "dozens of [[http://backstage.soundcloud.com/2012/07/go-at-soundcloud/][systems in Go]], touching almost every part of the site, and in many cases powering features from top to bottom." Engineer Peter Bourgon said "Go demonstrates that the cruft that burdens other languages and ecosystems—stuff that developers have learned to deal with, often in anger—is simply not a necessary part of modern programming. With Go, I have a straightforward and non-adversarial relationship with my tools, from development to production." - -- The [[https://ngrok.com/][ngrok]] service allows web developers to provide remote access to their development environments. Its author Alan Shreve said that "ngrok's success as a project is due in no small part to choosing Go as the implementation language," citing Go's HTTP libraries, efficiency, cross-platform compatibility, and ease of deployment as the major benefits. - -- [[http://poptip.com][Poptip]] provides social analytics services, and product engineer Andy Bonventre said "What started as an experiment in writing a single service in Go turned into moving almost our entire infrastructure over to it. What I love about Go the most is not necessarily the features of the language, but the focus on tooling, testing, and other elements that make writing large applications much more manageable." - -- Music collaboration startup [[http://splice.com][Splice]] chose to build their service with Go. Co-founder Matt Aimonetti said "We seriously studied and considered many programming languages, but Go's simplicity, efficiency, philosophy and community won us over." - -- And, of course, engineering teams across Google are moving to Go. Engineer Matt Welsh recently [[http://matt-welsh.blogspot.com.au/2013/08/rewriting-large-production-system-in-go.html][shared his experience]] rewriting a large production service in Go. Other notable public examples include YouTube's [[https://github.com/youtube/vitess][vitess project]] and [[http://talks.golang.org/2013/oscon-dl.slide][dl.google.com]]. We hope to share more stories like these soon. - -In September 2012, [[http://apcera.com/][Apcera]] CEO Derek Collison [[https://twitter.com/derekcollison/status/245522124666716160][predicted]] that "Go will become the dominant language for systems work in [Infastructure-as-a-Service], Orchestration, and [Platform-as-a-Service] in 24 months." Looking at the list above, it's easy to believe that prediction. - -So how can you get involved? Whether you're a seasoned Go programmer or just Go-curious, there are many ways to get started in the Go community: - -- [[http://blog.golang.org/getthee-to-go-meetup][Join your nearest Go User Group]], where your local gophers meet to share their knowledge and experience. These groups are popping up all over the world. I have personally spoken at Go groups in Amsterdam, Berlin, Gothenburg, London, Moscow, Munich, New York City, Paris, San Francisco, Seoul, Stockholm, Sydney, Tokyo, and Warsaw; but there are [[https://golang.org/wiki/GoUserGroups][many more]]! - -- Create or contribute to an open source Go project (or [[http://golang.org/doc/contribute.html][to Go itself]]). (And if you're building something, we'd love to hear from you on the [[http://groups.google.com/group/golang-nuts][Go mailing list]].) - -- If you're in Europe in February 2014, come along to the [[https://code.google.com/p/go-wiki/wiki/Fosdem2014][Go Devroom]] at [[https://fosdem.org/2014/][FOSDEM 2014]]. - -- Attend [[http://gophercon.com][GopherCon]], the first major Go conference, in Denver in April 2014. The event is organized by the [[http://www.gopheracademy.com][Gopher Academy]], who also run a [[http://www.gopheracademy.com/jobs][Go job board]]. - -The Go team has been amazed by the growth of the Go community over the past -four years. We are thrilled to see so many great things being built with Go, -and deeply grateful to work with our wonderful and dedicated contributors. -Thank you, everyone. - -Here's to four more years! diff --git a/content/5years.article b/content/5years.article deleted file mode 100644 index 42a2fa54..00000000 --- a/content/5years.article +++ /dev/null @@ -1,103 +0,0 @@ -Half a decade with Go -10 Nov 2014 - -Andrew Gerrand -adg@golang.org - -* Introduction - -Five years ago we launched the Go project. It seems like only yesterday that we -were preparing the initial public release: our -[[https://web.archive.org/web/20091112094121/http://golang.org/][website]] was -a lovely shade of yellow, we were calling Go a "systems language", and you had -to terminate statements with a semicolon and write Makefiles to build your -code. We had no idea how Go would be received. Would people share our vision -and goals? Would people find Go useful? - -At launch, there was a flurry of attention. Google had produced a new -programming language, and everyone was eager to check it out. Some programmers -were turned off by Go's conservative feature set—at first glance they saw -"nothing to see here"—but a smaller group saw the beginnings of an ecosystem -tailored to their needs as working software engineers. These few would form the -kernel of the Go community. - -.image 5years/gophers5th.jpg _ 850 - -[[/gopher][_Gopher_]] _illustration_by_ [[http://reneefrench.blogspot.com.au/][_Renee_French_]] - -After the initial release, it took us a while to properly communicate the -goals and design ethos behind Go. Rob Pike did so eloquently in his 2012 essay -[[http://talks.golang.org/2012/splash.article][_Go_at_Google:_Language_Design_in_the_Service_of_Software_Engineering_]] and -more personally in his blog post -[[http://commandcenter.blogspot.com.au/2012/06/less-is-exponentially-more.html][_Less_is_exponentially_more_]]. -Andrew Gerrand's -[[http://vimeo.com/53221560][_Code_that_grows_with_grace_]] -([[http://talks.golang.org/2012/chat.slide][slides]]) and -[[https://www.youtube.com/watch?v=dKGmK_Z1Zl0][_Go_for_Gophers_]] -([[http://talks.golang.org/2014/go4gophers.slide][slides]]) give a -more in-depth, technical take on Go's design philosophy. - -Over time, the few became many. The turning point for the project was the -release of Go 1 in March 2012, which provided a stable language and standard -library that developers could trust. By 2014, the project had hundreds of core -contributors, the ecosystem had countless [[https://godoc.org/][libraries and tools]] -maintained by thousands of developers, and the greater community had -many passionate members (or, as we call them, "gophers"). Today, by our current -metrics, the Go community is growing faster than we believed possible. - -Where can those gophers be found? They are at the many Go events that are -popping up around the world. This year we saw several dedicated Go conferences: -the inaugural [[http://blog.golang.org/gophercon][GopherCon]] and -[[http://www.dotgo.eu/][dotGo]] conferences in Denver and Paris, the -[[http://blog.golang.org/fosdem14][Go DevRoom at FOSDEM]] and two more -instances of the biannual [[https://github.com/GoCon/GoCon][GoCon]] conference -in Tokyo. At each event, gophers from around the globe eagerly presented their -Go projects. For the Go team, it is very satisfying to meet so many programmers -that share our vision and excitement. - -.image 5years/conferences.jpg - -_More_than_1,200_gophers_attended_GopherCon_in_Denver_and_dotGo_in_Paris._ - -There are also dozens of community-run -[[http://golang.org/wiki/GoUserGroups][Go User Groups]] spread across cities -worldwide. If you haven't visited your local group, consider going along. And -if there isn't a group in your area, maybe you should -[[https://blog.golang.org/getthee-to-go-meetup][start one]]? - -Today, Go has found a home in the cloud. Go arrived as the industry underwent a -tectonic shift toward cloud computing, and we were thrilled to see it quickly -become an important part of that movement. Its simplicity, efficiency, built-in -concurrency primitives, and modern standard library make it a great fit for -cloud software development (after all, that's what it was designed for). -Significant open source cloud projects like -[[https://www.docker.com/][Docker]] and -[[https://github.com/GoogleCloudPlatform/kubernetes][Kubernetes]] have been -written in Go, and infrastructure companies like Google, CloudFlare, Canonical, -Digital Ocean, GitHub, Heroku, and Microsoft are now using Go to do some heavy -lifting. - -So, what does the future hold? We think that 2015 will be Go's biggest year yet. - -Go 1.4—in addition to its [[http://tip.golang.org/doc/go1.4][new features and fixes]]—lays -the groundwork for a new low-latency garbage collector and support for running -Go on mobile devices. It is due to be released on December 1st 2014. -We expect the new GC to be available in Go 1.5, due June 1st 2015, which will -make Go appealing for a broader range of applications. -We can't wait to see where people take it. - -And there will be more great events, with [[http://gothamgo.com/][GothamGo]] in -New York (15 Nov), another Go DevRoom at FOSDEM in Brussels (Jan 31 and Feb 1; -[[https://groups.google.com/d/msg/golang-nuts/1xgBazQzs1I/hwrZ5ni8cTEJ][get involved!]]), -[[http://www.gophercon.in/][GopherCon India]] in Bengaluru (19-21 Feb), -the original [[http://gophercon.com/][GopherCon]] back at Denver in July, and -[[http://www.dotgo.eu/][dotGo]] on again at Paris in November. - -The Go team would like to extend its thanks to all the gophers out there. -Here's to the next five years. - -_To_celebrate_5_years_of_Go,_over_the_coming_month_the_ -[[http://blog.gopheracademy.com/][_Gopher_Academy_]] -_will_publish_a_series_of_articles_by_prominent_Go_users._Be_sure_to_check_out_ -[[http://blog.gopheracademy.com/][_their_blog_]] -_for_more_Go_action._ diff --git a/content/5years/conferences.jpg b/content/5years/conferences.jpg deleted file mode 100644 index 55962bc1..00000000 Binary files a/content/5years/conferences.jpg and /dev/null differ diff --git a/content/5years/gophers5th.jpg b/content/5years/gophers5th.jpg deleted file mode 100644 index df106486..00000000 Binary files a/content/5years/gophers5th.jpg and /dev/null differ diff --git a/content/6years-gopher.png b/content/6years-gopher.png deleted file mode 100644 index 71575637..00000000 Binary files a/content/6years-gopher.png and /dev/null differ diff --git a/content/6years.article b/content/6years.article deleted file mode 100644 index 6e4730ba..00000000 --- a/content/6years.article +++ /dev/null @@ -1,50 +0,0 @@ -Six years of Go -10 Nov 2015 - -Andrew Gerrand -adg@golang.org - -* Six years of Go - -Six years ago today the Go language was released as an open source project. -Since then, more than 780 contributors have made over 30,000 commits to the -project's 22 repositories. The ecosystem continues to grow, with GitHub -reporting more than 90,000 Go repositories. And, offline, we see new Go events -and user groups pop up [[https://blog.golang.org/gophercon2015][around]] -[[http://blog.golang.org/gouk15][the]] -[[http://blog.golang.org/gopherchina][world]] with regularity. - -.image 6years-gopher.png - -In August we [[https://blog.golang.org/go1.5][released Go 1.5]], the most -significant release since Go 1. It features a completely -[[https://golang.org/doc/go1.5#gc][redesigned garbage collector]] that makes -the language more suitable for latency-sensitive applications; it marks the -transition from a C-based compiler tool chain to one -[[https://golang.org/doc/go1.5#c][written entirely in Go]]; and it includes -ports to [[https://golang.org/doc/go1.5#ports][new architectures]], with better -support for ARM processors (the chips that power most smartphones). -These improvements make Go better suited to a broader range of tasks, a trend -that we hope will continue over the coming years. - -Improvements to tools continue to boost developer productivity. -We introduced the [[https://golang.org/cmd/trace/][execution tracer]] and the -"[[https://golang.org/cmd/go/#hdr-Show_documentation_for_package_or_symbol][go doc]]" -command, as well as more enhancements to our various -[[https://talks.golang.org/2014/static-analysis.slide][static analysis tools]]. -We are also working on an -[[https://groups.google.com/forum/#!topic/Golang-nuts/8oCSjAiKXUQ][official Go plugin for Sublime Text]], -with better support for other editors in the pipeline. - -Early next year we will release more improvements in Go 1.6, including -HTTP/2 support for [[https://golang.org/pkg/net/http/][net/http]] servers and -clients, an official package vendoring mechanism, support for blocks in text -and HTML templates, a memory sanitizer that checks both Go and C/C++ code, and -the usual assortment of other improvements and fixes. - -This is the sixth time we have had the pleasure of writing a birthday blog post -for Go, and we would not be doing so if not for the wonderful and passionate -people in our community. The Go team would like to thank everyone who has -contributed code, written an open source library, authored a blog post, helped -a new gopher, or just given Go a try. Without you, Go would not be as complete, -useful, or successful as it is today. Thank you, and celebrate! diff --git a/content/a-conversation-with-the-go-team.article b/content/a-conversation-with-the-go-team.article deleted file mode 100644 index 98d46124..00000000 --- a/content/a-conversation-with-the-go-team.article +++ /dev/null @@ -1,158 +0,0 @@ -A conversation with the Go team -6 Jun 2013 - -* Introduction - -At Google I/O 2013, several members of the Go team hosted a "Fireside chat." -Robert Griesemer, Rob Pike, David Symonds, Andrew Gerrand, Ian Lance Taylor, -Sameer Ajmani, Brad Fitzpatrick, and Nigel Tao took questions from the audience -and people around the world about various aspects of the Go project. - -.iframe http://www.youtube.com/embed/p9VUCp98ay4 309 549 - -We also hosted a similar session at I/O last year: -[[http://www.youtube.com/watch?v=sln-gJaURzk][_Meet_the_Go_team_]]. - -There were many more questions from Google Moderator than we were able to -answer in the short 40 minute session. -Here we answer some of those we missed in the live session. - -_Linking_speed_(and_memory_usage)_for_the_gc_toolchain_are_a_known_problem._ -_Are_there_any_plans_to_address_this_during_the_1.2_cycle?_ - -*Rob:* Yes. We are always thinking about ways to improve performance of the -tools as well as the language and libraries. - -_I_have_been_very_pleased_to_see_how_quickly_Go_appears_to_be_gaining_traction._ -_Can_you_talk_about_the_reactions_you_have_experienced_working_with_other_ -_developers_inside_and_outside_Google?_Are_there_any_major_sticking_points_remaining?_ - -*Robert:* A lot of developers that seriously tried Go are very happy with it. -Many of them report a much smaller, more readable and thus maintainable code -base: A 50% code size reduction or more when coming from C++ seems common. -Developers that switched to Go from Python are invariably pleased with the -performance gain. The typical complaints are about small inconsistencies in the -language (some of which we might iron out at some point). What surprises me is -that almost nobody complains about the lack of generics. - -_When_will_Go_be_a_first-class_language_for_Android_development?_ - -*Andrew:* This would be great, but we don't have anything to announce. - -_Is_there_a_roadmap_for_the_next_version_of_Go?_ - -*Andrew:* We have no feature roadmap as such. The contributors tend to work on -what interests them. Active areas of development include the gc and gccgo -compilers, the garbage collector and runtime, and many others. We expect the -majority of exciting new additions will be in the form of improvements to our -tools. You can find design discussions and code reviews on the -[[http://groups.google.com/group/golang-dev][golang-dev mailing list]]. - -As for the timeline, we do have -[[https://docs.google.com/document/d/106hMEZj58L9nq9N9p7Zll_WKfo-oyZHFyI6MttuZmBU/edit?usp=sharing][concrete plans]]: -we expect to release Go 1.2 on December 1, 2013. - -_Where_do_you_guys_want_to_see_Go_used_externally?_ -_What_would_you_consider_a_big_win_for_Go_adoption_outside_Google?_ -_Where_do_you_think_Go_has_the_potential_to_make_a_significant_impact?_ - -*Rob:* Where Go is deployed is up to its users, not to us. We're happy to see -it gain traction anywhere it helps. It was designed with server-side software -in mind, and is showing promise there, but has also shown strengths in many -other areas and the story is really just beginning. There are many surprises to -come. - -*Ian:* It’s easier for startups to use Go, because they don’t have an -entrenched code base that they need to work with. So I see two future big wins -for Go. One would be a significant use of Go by an existing large software -company other than Google. Another would be a significant IPO or acquisition -of a startup that primarily uses Go. These are both indirect: clearly choice -of programming language is a very small factor in the success of a company. -But it would be another way to show that Go can be part of a successful -software system. - -_Have_you_thought_any_(more)_about_the_potential_of_dynamically_loading_ -_Go_packages_or_objects_and_how_it_could_work_in_Go?_ -_I_think_this_could_enable_some_really_interesting_and_expressive_constructs,_ -_especially_coupled_with_interfaces._ - -*Rob:* This is an active topic of discussion. We appreciate how powerful the -concept can be and hope we can find a way to implement it before too long. -There are serious challenges in the design approach to take and the need to -make it work portably. - -_There_was_a_discussion_a_while_ago_about_collecting_some_best-of-breed_ -`database/sql` _drivers_in_a_more_central_place._ -_Some_people_had_strong_opinions_to_the_contrary_though._ -_Where_is_ `database/sql` _and_its_drivers_going_in_the_next_year?_ - -*Brad:* While we could create an official subrepo (“go.db”) for database -drivers, we fear that would unduly bless certain drivers. At this point we’d -still rather see healthy competition between different drivers. The -[[https://golang.org/wiki/SQLDrivers][SQLDrivers wiki page]] -lists some good ones. - -The `database/sql` package didn’t get much attention for a while, due to lack of -drivers. Now that drivers exist, usage of the package is increasing and -correctness and performance bugs are now being reported (and fixed). Fixes will -continue, but no major changes to the interface of `database/sql` are planned. - There might be small extensions here and there as needed for performance or to -assist some drivers. - -_What_is_the_status_of_versioning?_ -_Is_importing_some_code_from_github_a_best_practice_recommended_by_the_Go_team?_ -_What_happens_when_we_publish_our_code_that_is_dependent_on_a_github_repo_and_ -_the_API_of_the_dependee_changes?_ - -*Ian:* This is frequently discussed on the mailing list. What we do internally -is take a snapshot of the imported code, and update that snapshot from time to -time. That way, our code base won't break unexpectedly if the API changes. -But we understand that that approach doesn’t work very well for people who are -themselves providing a library. We’re open to good suggestions in this area. -Remember that this is an aspect of the tools that surround the language rather -than the language itself; the place to fix this is in the tools, not the -language. - -_What_about_Go_and_Graphical_User_Interfaces?_ - -*Rob:* This is a subject close to my heart. Newsqueak, a very early precursor -language, was designed specifically for writing graphics programs (that's what -we used to call apps). The landscape has changed a lot but I think Go's -concurrency model has much to offer in the field of interactive graphics. - -*Andrew:* There are many -[[https://golang.org/wiki/Projects#Graphics_and_Audio][bindings for existing graphics libraries]] -out there, and a few Go-specific projects. One of the more promising ones is -[[https://github.com/skelterjohn/go.uik][go.uik]], but it's still in its early -days. I think there's a lot of potential for a great Go-specific UI toolkit for -writing native applications (consider handling user events by receiving from a -channel), but developing a production-quality package is a significant -undertaking. I have no doubt one will come in time. - -In the meantime, the web is the most broadly available platform for user -interfaces. Go provides great support for building web apps, albeit only on the -back end. - -_In_the_mailing_lists_Adam_Langley_has_stated_that_the_TLS_code_has_not_been_ -_reviewed_by_outside_groups,_and_thus_should_not_be_used_in_production._ -_Are_there_plans_to_have_the_code_reviewed?_ -_A_good_secure_implementation_of_concurrent_TLS_would_be_very_nice._ - -*Adam*: Cryptography is notoriously easy to botch in subtle and surprising ways -and I’m only human. I don’t feel that I can warrant that Go’s TLS code is -flawless and I wouldn’t want to misrepresent it. - -There are a couple of places where the code is known to have side-channel -issues: the RSA code is blinded but not constant time, elliptic curves other -than P-224 are not constant time and the Lucky13 attack might work. I hope to -address the latter two in the Go 1.2 timeframe with a constant-time P-256 -implementation and AES-GCM. - -Nobody has stepped forward to do a review of the TLS stack however and I’ve not -investigated whether we could get Matasano or the like to do it. That depends -on whether Google wishes to fund it. - -_What_do_you_think_about_ [[http://www.gophercon.com/][_GopherCon_2014_]]_?_ -_Does_anyone_from_the_team_plan_to_attend?_ - -*Andrew:* It's very exciting. I'm sure some of us will be there. diff --git a/content/advanced-go-concurrency-patterns.article b/content/advanced-go-concurrency-patterns.article deleted file mode 100644 index d17638da..00000000 --- a/content/advanced-go-concurrency-patterns.article +++ /dev/null @@ -1,15 +0,0 @@ -Advanced Go Concurrency Patterns -23 May 2013 -Tags: talk, video, concurrency - -Andrew Gerrand - -* Introduction - -At Google I/O a year ago Rob Pike presented [[http://talks.golang.org/2012/concurrency.slide][_Go_Concurrency_Patterns_]], an introduction to Go's concurrency model. Last week, at I/O 2013, Go team member Sameer Ajmani continued the story with [[http://talks.golang.org/2013/advconc.slide][_Advanced_Go_Concurrency_Patterns_]], an in-depth look at a real concurrent programming problem. The talk shows how to detect and avoid deadlocks and race conditions, and demonstrates the implementation of deadlines, cancellation, and more. For those who want to take their Go programming to the next level, this is a must-see. - -.iframe http://www.youtube.com/embed/QDDwwePbDtw?rel=0 309 549 - -The slides are [[http://talks.golang.org/2013/advconc.slide][available here]] (use the left and right arrows to navigate). - -The slides were produced with [[http://godoc.org/golang.org/x/tools/present][the present tool]], and the runnable code snippets are powered by the [[http://play.golang.org/][Go Playground]]. The source code for this talk is in [[https://github.com/golang/talks/blob/master/2013/advconc.slide][the go.talks sub-repository]]. diff --git a/content/appengine-dec2013.article b/content/appengine-dec2013.article deleted file mode 100644 index db2e5de8..00000000 --- a/content/appengine-dec2013.article +++ /dev/null @@ -1,110 +0,0 @@ -Go on App Engine: tools, tests, and concurrency -13 Dec 2013 -Tags: appengine - -Andrew Gerrand - -Johan Euphrosine - -* Background - -When we [[http://blog.golang.org/go-and-google-app-engine][launched Go for App Engine]] -in May 2011 the SDK was just a modified version of the Python SDK. -At the time, there was no canonical way to build or organize Go programs, so it -made sense to take the Python approach. Since then Go 1.0 was released, -including the [[http://golang.org/cmd/go/][go tool]] and a -[[http://golang.org/doc/code.html][convention]] for organizing Go programs. - -In January 2013 we announced -[[http://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath][better integration]] -between the Go App Engine SDK and the go tool, promoting the use of -conventional import paths in App Engine apps and making it possible to use "go -get" to fetch app dependencies. - -With the recent release of App Engine 1.8.8 we are pleased to announce more -improvements to the developer experience for Go on App Engine. - -* The goapp tool - -The Go App Engine SDK now includes the "goapp" tool, an App Engine-specific -version of the "go" tool. The new name permits users to keep both the regular -"go" tool and the "goapp" tool in their system PATH. - -In addition to the existing "go" tool [[http://golang.org/cmd/go/][commands]], -the "goapp" tool provides new commands for working with App Engine apps. -The "[[https://developers.google.com/appengine/docs/go/tools/devserver][goapp serve]]" -command starts the local development server and the -"[[https://developers.google.com/appengine/docs/go/tools/uploadinganapp][goapp deploy]]" -command uploads an app to App Engine. - -The main advantages offered by the "goapp serve" and "goapp deploy" commands -are a simplified user interface and consistency with existing commands like -"go get" and "go fmt". -For example, to run a local instance of the app in the current directory, run: - - $ goapp serve - -To upload it to App Engine: - - $ goapp deploy - -You can also specify the Go import path to serve or deploy: - - $ goapp serve github.com/user/myapp - -You can even specify a YAML file to serve or deploy a specific -[[https://developers.google.com/appengine/docs/go/modules/][module]]: - - $ goapp deploy mymodule.yaml - -These commands can replace most uses of `dev_appserver.py` and `appcfg.py`, -although the Python tools are still available for their less common uses. - -* Local unit testing - -The Go App Engine SDK now supports local unit testing, using Go's native -[[https://developers.google.com/appengine/docs/go/tools/localunittesting][testing package]] -and the "[[http://golang.org/cmd/go/#hdr-Test_packages][go test]]" command -(provided as "goapp test" by the SDK). - -Furthermore, you can now write tests that use App Engine services. -The [[https://developers.google.com/appengine/docs/go/tools/localunittesting#Go_Introducing_the_aetest_package][aetest package]] -provides an appengine.Context value that delegates requests to a temporary -instance of the development server. - -For more information about using "goapp test" and the aetest package, see the -[[https://developers.google.com/appengine/docs/go/tools/localunittesting][Local Unit Testing for Go documentation]]. -Note that the aetest package is still in its early days; -we hope to add more features over time. - -* Better concurrency support - -It is now possible to configure the number of concurrent requests served by -each of your app's dynamic instances by setting the -[[https://developers.google.com/appengine/docs/go/modules/#max_concurrent_requests][`max_concurrent_requests`]] option -(available to [[https://developers.google.com/appengine/docs/go/modules/#automatic_scaling][Automatic Scaling modules]] only). - -Here's an example `app.yaml` file: - - application: maxigopher - version: 1 - runtime: go - api_version: go1 - automatic_scaling: - max_concurrent_requests: 100 - -This configures each instance of the app to serve up to 100 requests -concurrently (up from the default of 10). You can configure Go instances to -serve up to a maximum of 500 concurrent requests. - -This setting allows your instances to handle more simultaneous requests by -taking advantage of Go's efficient handling of concurrency, which should yield -better instance utilization and ultimately fewer billable instance hours. - -* Conclusion - -With these changes Go on App Engine is more convenient and efficient than ever, -and we hope you enjoy the improvements. Please join the -[[http://groups.google.com/group/google-appengine-go/][google-appengine-go group]] -to raise questions or discuss these changes with the engineering team and the -rest of the community. diff --git a/content/building-stathat-with-go.article b/content/building-stathat-with-go.article deleted file mode 100644 index e9d0d25b..00000000 --- a/content/building-stathat-with-go.article +++ /dev/null @@ -1,109 +0,0 @@ -Building StatHat with Go -19 Dec 2011 -Tags: guest - -Patrick Crosby - -* Introduction - -My name is Patrick Crosby and I'm the founder of a company called Numerotron. We recently released [[http://www.stathat.com][StatHat]]. This post is about why we chose to develop StatHat in [[http://golang.org][Go]], including details about how we are using Go. - -[[http://www.stathat.com][StatHat]] is a tool to track statistics and events in your code. Everyone from HTML designers to backend engineers can use StatHat easily, as it supports sending stats from HTML, JavaScript, Go, and twelve other languages. - -You send your numbers to StatHat; it generates beautiful, fully-embeddable graphs of your data. StatHat will alert you when specified triggers occur, send you daily email reports, and much more. So instead of spending time writing tracking or reporting tools for your application, you can concentrate on the code. While you do the real work, StatHat remains intensely vigilant, like an eagle in its mountaintop nest, or a babysitter on meth. - -Here's an example of a StatHat graph of the temperature in NYC, Chicago, and San Francisco: - -.image building-stathat-with-go_weather.png - -* Architecture Overview - -StatHat consists of two main services: incoming statistic/event API calls and the web application for viewing and analyzing stats. We wanted to keep these as separate as possible to isolate the data collection from the data interaction. We did this for many reasons, but one major reason is that we anticipate handling a ton of automated incoming API HTTP requests and would thus have different optimization strategies for the API service than a web application interacting with humans. - -.image building-stathat-with-go_stathat_architecture.png - -The web application service is multi-tiered. The web server processes all requests and sends them to an interactor layer. For simple tasks, the interactor will handle generating any necessary data. For complex tasks, the interactor relies on multiple application servers to handle tasks like generating graphs or analyzing data sets. After the interactor is finished, the web server sends the result to a presenter. The presenter responds to the HTTP request with either HTML or JSON. We can horizontally scale the web, API, application servers, and databases as the demand for services grows and changes over time. There is no single point of failure as each application server has multiple copies running. The interactor layer allows us to have different interfaces to the system: http, command line, automated tests, mobile API. StatHat uses MySQL for data storage. - -* Choosing Go - -When we designed StatHat, we had the following check list for our development tools: - -- same programming language for backend and frontend systems - -- good, fast HTML templating system - -- fast start-up, recompilation, testing for lots of tinkering - -- lots of connections on one machine - -- language tools for handling application-level concurrency - -- good performance - -- robust RPC layer to talk between tiers - -- lots of libraries - -- open source - -We evaluated many popular and not-so-popular web technologies and ended up choosing to develop it in Go. - -When Go was released in November 2009, I immediately installed it and loved the fast compilation times, goroutines, channels, garbage collection, and all the packages that were available. I was especially pleased with how few lines of code my applications were using. I soon experimented with making a web app called [[http://langalot.com/][Langalot]] that concurrently searched through five foreign language dictionaries as you typed in a query. It was blazingly fast. I put it online and it's been running since February, 2010. - -The following sections detail how Go meets StatHat's requirements and our experience using Go to solve our problems. - -* Runtime - -We use the standard Go [[http://golang.org/pkg/http/][http package]] for our API and web app servers. All requests first go through Nginx and any non-file requests are proxied to the Go-powered http servers. The backend servers are all written in Go and use the [[http://golang.org/pkg/rpc/][rpc package]] to communicate with the frontend. - -* Templating - -We built a template system using the standard [[http://golang.org/pkg/template/][template package]]. Our system adds layouts, some common formatting functions, and the ability to recompile templates on-the-fly during development. We are very pleased with the performance and functionality of the Go templates. - -* Tinkering - -In a previous job, I worked on a video game called Throne of Darkness that was written in C++. We had a few header files that, when modified, required a full rebuild of the entire system, 20-30 minutes long. If anyone ever changed `Character.h`, he would be subject to the wrath of every other programmer. Besides this suffering, it also slowed down development time significantly. - -Since then, I've always tried to choose technologies that allowed fast, frequent tinkering. With Go, compilation time is a non-issue. We can recompile the entire system in seconds, not minutes. The development web server starts instantly, tests complete in a few seconds. As mentioned previously, templates are recompiled as they change. The result is that the StatHat system is very easy to work with, and the compiler is not a bottleneck. - -* RPC - -Since StatHat is a multi-tiered system, we wanted an RPC layer so that all communication was standard. With Go, we are using the [[http://golang.org/pkg/rpc/][rpc package]] and the [[http://golang.org/pkg/gob/][gob package]] for encoding Go objects. In Go, the RPC server just takes any Go object and registers its exported methods. There is no need for an intermediary interface description language. We've found it very easy to use and many of our core application servers are under 300 lines of code. - -* Libraries - -We don't want to spend time rewriting libraries for things like SSL, database drivers, JSON/XML parsers. Although Go is a young language, it has a lot of system packages and a growing number of user-contributed packages. With only a few exceptions, we have found Go packages for everything we have needed. - -* Open source - -In our experience, it has been invaluable to work with open source tools. If something is going awry, it is immensely helpful to be able to examine the source through every layer and not have any black boxes. Having the code for the language, web server, packages, and tools allows us to understand how every piece of the system works. Everything in Go is open source. In the Go codebase, we frequently read the tests as they often give great examples of how to use packages and language features. - -* Performance - -People rely on StatHat for up to the minute analysis of their data and we need the system to be as responsive as possible. In our tests, Go's performance blew away most of the competition. We tested it against Rails, Sinatra, OpenResty, and Node. StatHat has always monitored itself by tracking all kinds of performance metrics about requests, the duration of certain tasks, the amount of memory in use. Because of this, we were able to easily evaluate different technologies. We've also taken advantage of the benchmark performance testing features of the Go testing package. - -* Application-Level Concurrency - -In a former life, I was the CTO at OkCupid. My experience there using OKWS taught me the importance of async programming, especially when it comes to dynamic web applications. There is no reason you should ever do something like this synchronously: load a user from the database, then find their stats, then find their alerts. These should all be done concurrently, yet surprisingly, many popular frameworks have no async support. Go supports this at the language level without any callback spaghetti. StatHat uses goroutines extensively to run multiple functions concurrently and channels for sharing data between goroutines. - -* Hosting and Deployment - -StatHat runs on Amazon's EC2 servers. Our servers are divided into several types: - -- API - -- Web - -- Application servers - -- Database - -There are at least two of each type of server, and they are in different zones for high availability. Adding a new server to the mix takes just a couple of minutes. - -To deploy, we first build the entire system into a time-stamped directory. Our packaging script builds the Go applications, compresses the CSS and JS files, and copies all the scripts and configuration files. This directory is then distributed to all the servers, so they all have an identical distribution. A script on each server queries its EC2 tags and determines what it is responsible for running and starts/stops/restarts any services. We frequently only deploy to a subset of the servers. - -* More - -For more information on StatHat, please visit [[http://www.stathat.com][stathat.com]]. We are releasing some of the Go code we've written. Go to [[http://www.stathat.com/src][www.stathat.com/src]] for all of the open source StatHat projects. - -To learn more about Go, visit [[http://golang.org/][golang.org]]. diff --git a/content/building-stathat-with-go_stathat_architecture.png b/content/building-stathat-with-go_stathat_architecture.png deleted file mode 100644 index d07adc7d..00000000 Binary files a/content/building-stathat-with-go_stathat_architecture.png and /dev/null differ diff --git a/content/building-stathat-with-go_weather.png b/content/building-stathat-with-go_weather.png deleted file mode 100644 index f03f16a2..00000000 Binary files a/content/building-stathat-with-go_weather.png and /dev/null differ diff --git a/content/c-go-cgo.article b/content/c-go-cgo.article deleted file mode 100644 index d0e9976c..00000000 --- a/content/c-go-cgo.article +++ /dev/null @@ -1,109 +0,0 @@ -C? Go? Cgo! -17 Mar 2011 -Tags: cgo, technical - -Andrew Gerrand - -* Introduction - -Cgo lets Go packages call C code. Given a Go source file written with some special features, cgo outputs Go and C files that can be combined into a single Go package. - -To lead with an example, here's a Go package that provides two functions - `Random` and `Seed` - that wrap C's `random` and `srandom` functions. - - package rand - - /* - #include - */ - import "C" - - func Random() int { - return int(C.random()) - } - - func Seed(i int) { - C.srandom(C.uint(i)) - } - -Let's look at what's happening here, starting with the import statement. - -The `rand` package imports `"C"`, but you'll find there's no such package in the standard Go library. That's because `C` is a "pseudo-package", a special name interpreted by cgo as a reference to C's name space. - -The `rand` package contains four references to the `C` package: the calls to `C.random` and `C.srandom`, the conversion `C.uint(i)`, and the `import` statement. - -The `Random` function calls the standard C library's `random` function and returns the result. In C, `random` returns a value of the C type `long`, which cgo represents as the type `C.long`. It must be converted to a Go type before it can be used by Go code outside this package, using an ordinary Go type conversion: - - func Random() int { - return int(C.random()) - } - -Here's an equivalent function that uses a temporary variable to illustrate the type conversion more explicitly: - - func Random() int { - var r C.long = C.random() - return int(r) - } - -The `Seed` function does the reverse, in a way. It takes a regular Go `int`, converts it to the C `unsigned`int` type, and passes it to the C function `srandom`. - - func Seed(i int) { - C.srandom(C.uint(i)) - } - -Note that cgo knows the `unsigned`int` type as `C.uint`; see the [[http://golang.org/cmd/cgo][cgo documentation]] for a complete list of these numeric type names. - -The one detail of this example we haven't examined yet is the comment above the `import` statement. - - /* - #include - */ - import "C" - -Cgo recognizes this comment. Any lines starting with `#cgo` followed by a space character are removed; these become directives for cgo. The remaining lines are used as a header when compiling the C parts of the package. In this case those lines are just a single `#include` statement, but they can be almost any C code. The `#cgo` directives are used to provide flags for the compiler and linker when building the C parts of the package. - -There is a limitation: if your program uses any `//export` directives, then the C code in the comment may only include declarations (`extern`int`f();`), not definitions (`int`f()`{`return`1;`}`). You can use `//export` directives to make Go functions accessible to C code. - -The `#cgo` and `//export` directives are documented in the [[http://golang.org/cmd/cgo/][cgo documentation]]. - -* Strings and things - -Unlike Go, C doesn't have an explicit string type. Strings in C are represented by a zero-terminated array of chars. - -Conversion between Go and C strings is done with the `C.CString`, `C.GoString`, and `C.GoStringN` functions. These conversions make a copy of the string data. - -This next example implements a `Print` function that writes a string to standard output using C's `fputs` function from the `stdio` library: - - package print - - // #include - // #include - import "C" - import "unsafe" - - func Print(s string) { - cs := C.CString(s) - C.fputs(cs, (*C.FILE)(C.stdout)) - C.free(unsafe.Pointer(cs)) - } - -Memory allocations made by C code are not known to Go's memory manager. When you create a C string with `C.CString` (or any C memory allocation) you must remember to free the memory when you're done with it by calling `C.free`. - -The call to `C.CString` returns a pointer to the start of the char array, so before the function exits we convert it to an [[http://golang.org/pkg/unsafe/#Pointer][`unsafe.Pointer`]] and release the memory allocation with `C.free`. A common idiom in cgo programs is to [[http://golang.org/doc/articles/defer_panic_recover.html][`defer`]] the free immediately after allocating (especially when the code that follows is more complex than a single function call), as in this rewrite of `Print`: - - func Print(s string) { - cs := C.CString(s) - defer C.free(unsafe.Pointer(cs)) - C.fputs(cs, (*C.FILE)(C.stdout)) - } - -* Building cgo packages - -To build cgo packages, just use [[http://golang.org/cmd/go/#Compile_packages_and_dependencies][`go`build`]] or [[http://golang.org/cmd/go/#Compile_and_install_packages_and_dependencies][`go`install`]] as usual. The go tool recognizes the special `"C"` import and automatically uses cgo for those files. - -* More cgo resources - -The [[http://golang.org/cmd/cgo/][cgo command]] documentation has more detail about the C pseudo-package and the build process. The [[http://golang.org/misc/cgo/][cgo examples]] in the Go tree demonstrate more advanced concepts. - -For a simple, idiomatic example of a cgo-based package, see Russ Cox's [[http://code.google.com/p/gosqlite/source/browse/sqlite/sqlite.go][gosqlite]]. Also, the Go Project Dashboard lists [[https://godashboard.appspot.com/project?tag=cgo][several other cgo packages]]. - -Finally, if you're curious as to how all this works internally, take a look at the introductory comment of the runtime package's [[https://golang.org/src/runtime/cgocall.go][cgocall.go]]. diff --git a/content/concurrency-is-not-parallelism.article b/content/concurrency-is-not-parallelism.article deleted file mode 100644 index c3ff6118..00000000 --- a/content/concurrency-is-not-parallelism.article +++ /dev/null @@ -1,19 +0,0 @@ -Concurrency is not parallelism -16 Jan 2013 -Tags: concurrency, talk, video - -Andrew Gerrand - -* Introduction - -If there's one thing most people know about Go, is that it is designed for concurrency. No introduction to Go is complete without a demonstration of its goroutines and channels. - -But when people hear the word _concurrency_ they often think of _parallelism_, a related but quite distinct concept. In programming, concurrency is the _composition_ of independently executing processes, while parallelism is the simultaneous _execution_ of (possibly related) computations. Concurrency is about _dealing_with_ lots of things at once. Parallelism is about _doing_ lots of things at once. - -To clear up this conflation, Rob Pike gave a talk at [[http://heroku.com/][Heroku]]'s [[http://waza.heroku.com/][Waza]] conference entitled _Concurrency_is_not_parallelism_, and a video recording of the talk was released a few months ago. - -.iframe http://player.vimeo.com/video/49718712?badge=0 281 500 - -The slides are available at [[http://talks.golang.org/2012/waza.slide][talks.golang.org]] (use the left and right arrow keys to navigate). - -To learn about Go's concurrency primitives, watch [[http://www.youtube.com/watch?v=f6kdp27TYZs][Go concurrency patterns]] ([[http://talks.golang.org/2012/concurrency.slide][slides]]). diff --git a/content/constants.article b/content/constants.article deleted file mode 100644 index 6393204a..00000000 --- a/content/constants.article +++ /dev/null @@ -1,452 +0,0 @@ -Constants -25 Aug 2014 -Tags: constants - -Rob Pike - -* Introduction - -Go is a statically typed language that does not permit operations that mix numeric types. -You can't add a `float64` to an `int`, or even an `int32` to an `int`. -Yet it is legal to write `1e6*time.Second` or `math.Exp(1)` or even `1<<('\t'+2.0)`. -In Go, constants, unlike variables, behave pretty much like regular numbers. -This post explains why that is and what it means. - -* Background: C - -In the early days of thinking about Go, we talked about a number of problems caused by the way C and its descendants let you mix and match numeric types. -Many mysterious bugs, crashes, and portability problems are caused by expressions that combine integers of different sizes and "signedness". -Although to a seasoned C programmer the result of a calculation like - - unsigned int u = 1e9; - long signed int i = -1; - ... i + u ... - -may be familiar, it isn't _a_priori_ obvious. -How big is the result? -What is its value? -Is it signed or unsigned? - -Nasty bugs lurk here. - -C has a set of rules called "the usual arithmetic conversions" and it is an indicator of their subtlety that they have changed over the years (introducing yet more bugs, retroactively). - -When designing Go, we decided to avoid this minefield by mandating that there is _no_ mixing of numeric types. -If you want to add `i` and `u`, you must be explicit about what you want the result to be. -Given - - var u uint - var i int - -you can write either `uint(i)+u` or `i+int(u)`, with both the meaning and type of the addition clearly expressed, but unlike in C you cannot write `i+u`. -You can't even mix `int` and `int32`, even when `int` is a 32-bit type. - -This strictness eliminates a common cause of bugs and other failures. -It is a vital property of Go. -But it has a cost: it sometimes requires programmers to decorate their code with clumsy numeric conversions to express their meaning clearly. - -And what about constants? -Given the declarations above, what would make it legal to write `i` `=` `0` or `u` `=` `0`? -What is the _type_ of `0`? -It would be unreasonable to require constants to have type conversions in simple contexts such as `i` `=` `int(0)`. - -We soon realized the answer lay in making numeric constants work differently from how they behave in other C-like languages. -After much thinking and experimentation, we came up with a design that we believe feels right almost always, freeing the programmer from converting constants all the time yet being able to write things like `math.Sqrt(2)` without being chided by the compiler. - -In short, constants in Go just work, most of the time anyway. -Let's see how that happens. - -* Terminology - -First, a quick definition. -In Go, `const` is a keyword introducing a name for a scalar value such as `2` or `3.14159` or `"scrumptious"`. -Such values, named or otherwise, are called _constants_ in Go. -Constants can also be created by expressions built from constants, such as `2+3` or `2+3i` or `math.Pi/2` or `("go"+"pher")`. - -Some languages don't have constants, and others have a more general definition of constant or application of the word `const`. -In C and C++, for instance, `const` is a type qualifier that can codify more intricate properties of more intricate values. - -But in Go, a constant is just a simple, unchanging value, and from here on we're talking only about Go. - -* String constants - -There are many kinds of numeric constants—integers, floats, runes, signed, unsigned, imaginary, complex—so let's start with a simpler form of constant: strings. -String constants are easy to understand and provide a smaller space in which to explore the type issues of constants in Go. - -A string constant encloses some text between double quotes. -(Go has also has raw string literals, enclosed by backquotes ``````, but for the purpose of this discussion they have all the same properties.) -Here is a string constant: - - "Hello, 世界" - -(For much more detail about the representation and interpretation of strings, see [[//blog.golang.org/strings][this blog post]].) - -What type does this string constant have? -The obvious answer is `string`, but that is _wrong_. - -This is an _untyped_string_constant_, which is to say it is a constant textual value that does not yet have a fixed type. -Yes, it's a string, but it's not a Go value of type `string`. -It remains an untyped string constant even when given a name: - - const hello = "Hello, 世界" - -After this declaration, `hello` is also an untyped string constant. -An untyped constant is just a value, one not yet given a defined type that would force it to obey the strict rules that prevent combining differently typed values. - -It is this notion of an _untyped_ constant that makes it possible for us to use constants in Go with great freedom. - -So what, then, is a _typed_ string constant? -It's one that's been given a type, like this: - - const typedHello string = "Hello, 世界" - -Notice that the declaration of `typedHello` has an explicit `string` type before the equals sign. -This means that `typedHello` has Go type `string`, and cannot be assigned to a Go variable of a different type. -That is to say, this code works: - -.play -edit constants/string1.go /START/,/STOP/ - -but this does not: - -.play -edit constants/string2.go /START/,/STOP/ - -The variable `m` has type `MyString` and cannot be assigned a value of a different type. -It can only be assigned values of type `MyString`, like this: - -.play -edit constants/string3.go /START/,/STOP/ - -or by forcing the issue with a conversion, like this: - -.play -edit constants/string4.go /START/,/STOP/ - -Returning to our _untyped_ string constant, it has the helpful property that, since it has no type, assigning it to a typed variable does not cause a type error. -That is, we can write - - m = "Hello, 世界" - -or - - m = hello - -because, unlike the typed constants `typedHello` and `myStringHello`, the untyped constants `"Hello,`世界"` and `hello` _have_no_type_. -Assigning them to a variable of any type compatible with strings works without error. - -These untyped string constants are strings, of course, so they can only be used where a string is allowed, but they do not have _type_ `string`. - -* Default type - -As a Go programmer, you have certainly seen many declarations like - - str := "Hello, 世界" - -and by now you might be asking, "if the constant is untyped, how does `str` get a type in this variable declaration?" -The answer is that an untyped constant has a default type, an implicit type that it transfers to a value if a type is needed where none is provided. -For untyped string constants, that default type is obviously `string`, so - - str := "Hello, 世界" - -or - - var str = "Hello, 世界" - -means exactly the same as - - var str string = "Hello, 世界" - -One way to think about untyped constants is that they live in a kind of ideal space of values, a space less restrictive than Go's full type system. -But to do anything with them, we need to assign them to variables, and when that happens the _variable_ (not the constant itself) needs a type, and the constant can tell the variable what type it should have. -In this example, `str` becomes a value of type `string` because the untyped string constant gives the declaration its default type, `string`. - -In such a declaration, a variable is declared with a type and initial value. -Sometimes when we use a constant, however, the destination of the value is not so clear. -For instance consider this statement: - -.play -edit constants/default1.go /START/,/STOP/ - -The signature of `fmt.Printf` is - - func Printf(format string, a ...interface{}) (n int, err error) - -which is to say its arguments (after the format string) are interface values. -What happens when `fmt.Printf` is called with an untyped constant is that an interface value is created -to pass as an argument, and the concrete type stored for that argument is the default type of the constant. -This process is analogous to what we saw earlier when declaring an initialized value using an untyped string constant. - -You can see the result in this example, which uses the format `%v` to print the value and `%T` to print the type of the value being passed to `fmt.Printf`: - -.play -edit constants/default2.go /START/,/STOP/ - -If the constant has a type, that goes into the interface, as this example shows: - -.play -edit constants/default3.go /START/,/STOP/ - -(For more information about how interface values work, see the first sections of [[//blog.golang.org/laws-of-reflection][this blog post]].) - -In summary, a typed constant obeys all the rules of typed values in Go. -On the other hand, an untyped constant does not carry a Go type in the same way and can be mixed and matched more freely. -It does, however, have a default type that is exposed when, and only when, no other type information is available. - -* Default type determined by syntax - -The default type of an untyped constant is determined by its syntax. -For string constants, the only possible implicit type is `string`. -For [[http://golang.org/ref/spec#Numeric_types][numeric constants]], the implicit type has more variety. -Integer constants default to `int`, floating-point constants `float64`, rune constants to `rune` (an alias for `int32`), and imaginary constants to `complex128`. -Here's our canonical print statement used repeatedly to show the default types in action: - -.play -edit constants/syntax.go /START/,/STOP/ - -(Exercise: Explain the result for `'x'`.) - -* Booleans - -Everything we said about untyped string constants can be said for untyped boolean constants. -The values `true` and `false` are untyped boolean constants that can be assigned to any boolean variable, -but once given a type, boolean variables cannot be mixed: - -.play -edit constants/bool.go /START/,/STOP/ - -Run the example and see what happens, then comment out the "Bad" line and run it again. -The pattern here follows exactly that of string constants. - -* Floats - -Floating-point constants are just like boolean constants in most respects. -Our standard example works as expected in translation: - -.play -edit constants/float1.go /START/,/STOP/ - -One wrinkle is that there are _two_ floating-point types in Go: `float32` and `float64`. -The default type for a floating-point constant is `float64`, although an untyped floating-point -constant can be assigned to a `float32` value just fine: - -.play -edit constants/float2.go /START/,/STOP/ - -Floating-point values are a good place to introduce the concept of overflow, or the range of values. - -Numeric constants live in an arbitrary-precision numeric space; they are just regular numbers. -But when they are assigned to a variable the value must be able to fit in the destination. -We can declare a constant with a very large value: - -.code constants/float3.go /Huge/ - -—that's just a number, after all—but we can't assign it or even print it. This statement won't even compile: - -.play -edit constants/float3.go /Println/ - -The error is, "constant 1.00000e+1000 overflows float64", which is true. -But `Huge` might be useful: we can use it in expressions with other constants and use the value of those expressions if the result -can be represented in the range of a `float64`. -The statement, - -.play -edit constants/float4.go /Println/ - -prints `10`, as one would expect. - -In a related way, floating-point constants may have very high precision, so that arithmetic involving them is more accurate. -The constants defined in the [[//golang.org/pkg/math][math]] package are given with many more digits than are -available in a `float64`. Here is the definition of `math.Pi`: - - Pi = 3.14159265358979323846264338327950288419716939937510582097494459 - -When that value is assigned to a variable, some of the precision will be lost; the assignment will create the `float64` (or `float32`) -value closest to the high-precision value. This snippet - -.play -edit constants/float5.go /START/,/STOP/ - -prints `3.141592653589793`. - -Having so many digits available means that calculations like `Pi/2` or other more intricate evaluations can carry more precision -until the result is assigned, making calculations involving constants easier to write without losing precision. -It also means that there is no occasion in which the floating-point corner cases like infinities, -soft underflows, and `NaNs` arise in constant expressions. -(Division by a constant zero is a compile-time error, and when everything is a number there's no such thing as "not a number".) - -* Complex numbers - -Complex constants behave a lot like floating-point constants. -Here's a version of our now-familiar litany translated into complex numbers: - -.play -edit constants/complex1.go /START/,/STOP/ - -The default type of a complex number is `complex128`, the larger-precision version composed of two `float64` values. - -For clarity in our example, we wrote out the full expression `(0.0+1.0i)`, but this value can be shortened to `0.0+1.0i`, -`1.0i` or even `1i`. - -Let's play a trick. -We know that in Go, a numeric constant is just a number. -What if that number is a complex number with no imaginary part, that is, a real? -Here's one: - -.code constants/complex2.go /const Two/ - -That's an untyped complex constant. -Even though it has no imaginary part, the _syntax_ of the expression defines it to have default type `complex128`. -Therefore, if we use it to declare a variable, the default type will be `complex128`. The snippet - -.play -edit constants/complex2.go /START/,/STOP/ - -prints `complex128:` `(2+0i)`. -But numerically, `Two` can be stored in a scalar floating-point number, a `float64` or `float32`, with no loss of information. -Thus we can assign `Two` to a `float64`, either in an initialization or an assignment, without problems: - -.play -edit constants/complex3.go /START/,/STOP/ - -The output is `2` `and` `2`. -Even though `Two` is a complex constant, it can be assigned to scalar floating-point variables. -This ability for a constant to "cross" types like this will prove useful. - -* Integers - -At last we come to integers. -They have more moving parts—[[http://golang.org/ref/spec#Numeric_types][many sizes, signed or unsigned, and more]]—but they play by the same rules. -For the last time, here is our familiar example, using just `int` this time: - -.play -edit constants/int1.go /START/,/STOP/ - -The same example could be built for any of the integer types, which are: - - int int8 int16 int32 int64 - uint uint8 uint16 uint32 uint64 - uintptr - -(plus the aliases `byte` for `uint8` and `rune` for `int32`). -That's a lot, but the pattern in the way constants work should be familiar enough by now that you can see how things will play out. - -As mentioned above, integers come in a couple of forms and each form has its own default type: `int` for simple constants like `123` or `0xFF` or `-14` -and `rune` for quoted characters like 'a', '世' or '\r'. - -No constant form has as its default type an unsigned integer type. -However, the flexibility of untyped constants means we can initialize unsigned integer variables using simple constants as long as we are clear about the type. -It's analogous to how we can initialize a `float64` using a complex number with zero imaginary part. -Here are several different ways to initialize a `uint`; all are equivalent, but all must mention the type explicitly for the result to be unsigned. - - var u uint = 17 - var u = uint(17) - u := uint(17) - -Similarly to the range issue mentioned in the section on floating-point values, not all integer values can fit in all integer types. -There are two problems that might arise: the value might be too large, or it might be a negative value being assigned to an unsigned integer type. -For instance, `int8` has range -128 through 127, so constants outside of that range can never be assigned to a variable of type `int8`: - -.play -edit constants/int2.go /var/ - -Similarly, `uint8`, also known as `byte`, has range 0 through 255, so a large or negative constant cannot be assigned to a `uint8`: - -.play -edit constants/int3.go /var/ - -This type-checking can catch mistakes like this one: - -.play -edit constants/int4.go /START/,/STOP/ - -If the compiler complains about your use of a constant, it's likely a real bug like this. - -* An exercise: The largest unsigned int - -Here is an informative little exercise. -How do we express a constant representing the largest value that fits in a `uint`? -If we were talking about `uint32` rather than `uint`, we could write - - const MaxUint32 = 1<<32 - 1 - -but we want `uint`, not `uint32`. -The `int` and `uint` types have equal unspecified numbers of bits, either 32 or 64. -Since the number of bits available depends on the architecture, we can't just write down a single value. - -Fans of [[http://en.wikipedia.org/wiki/Two's_complement][two's-complement arithmetic]], -which Go's integers are defined to use, know that the representation of `-1` has all its bits set to 1, -so the bit pattern of `-1` is internally the same as that of the -largest unsigned integer. -We therefore might think we could write - -.play -edit constants/exercise1.go /const/ - -but that is illegal because -1 cannot be represented by an unsigned variable; `-1` is not in the range of unsigned values. -A conversion won't help either, for the same reason: - -.play -edit constants/exercise2.go /const/ - -Even though at run-time a value of -1 can be converted to an unsigned integer, the rules -for constant [[http://golang.org/ref/spec#Conversions][conversions]] forbid this kind of coercion at compile time. -That is to say, this works: - -.play -edit constants/exercise3.go /START/,/STOP/ - -but only because `v` is a variable; if we made `v` a constant, even an untyped constant, we'd be back in forbidden territory: - -.play -edit constants/exercise4.go /START/,/STOP/ - -We return to our previous approach, but instead of `-1` we try `^0`, the bitwise negation of an arbitrary number of zero bits. -But that fails too, for a similar reason: -In the space of numeric values, -`^0` represents an infinite number of ones, so we lose information if we assign that to any fixed-size integer: - -.play -edit constants/exercise5.go /const/ - -How then do we represent the largest unsigned integer as a constant? - -The key is to constrain the operation to the number of bits in a `uint` and avoiding -values, such as negative numbers, that are not representable in a `uint`. -The simplest `uint` value is the typed constant `uint(0)`. -If `uints` have 32 or 64 bits, `uint(0)` has 32 or 64 zero bits accordingly. -If we invert each of those bits, we'll get the correct number of one bits, which is the largest `uint` value. - -Therefore we don't flip the bits of the untyped constant `0`, we flip the bits of the typed constant `uint(0)`. -Here, then, is our constant: - -.play -edit constants/exercise6.go /START/,/STOP/ - -Whatever the number of bits it takes to represent a `uint` in the current execution environment -(on the [[http://blog.golang.org/playground][playground]], it's 32), -this constant correctly represents the largest value a variable of type `uint` can hold. - -If you understand the analysis that got us to this result, you understand all the important points about constants in Go. - -* Numbers - -The concept of untyped constants in Go means that all the numeric constants, whether integer, floating-point, complex, or even character values, -live in a kind of unified space. -It's when we bring them to the computational world of variables, assignments, and operations that the actual types matter. -But as long as we stay in the world of numeric constants, we can mix and match values as we like. -All these constants have numeric value 1: - - 1 - 1.000 - 1e3-99.0*10-9 - '\x01' - '\u0001' - 'b' - 'a' - 1.0+3i-3.0i - -Therefore, although they have different implicit default types, written as untyped constants they can be assigned to a variable of any integer type: - -.play -edit constants/numbers1.go /START/,/STOP/ - -The output from this snippet is: `1`1`1`1`1`(1+0i)`1`. - -You can even do nutty stuff like - -.play -edit constants/numbers2.go /START/,/STOP/ - -which yields 145.5, which is pointless except to prove a point. - -But the real point of these rules is flexibility. -That flexibility means that, despite the fact that in Go it is illegal in the same expression to mix floating-point and integer variables, -or even `int` and `int32` variables, it is fine to write - - sqrt2 := math.Sqrt(2) - -or - - const millisecond = time.Second/1e3 - -or - - bigBufferWithHeader := make([]byte, 512+1e6) - -and have the results mean what you expect. - -Because in Go, numeric constants work as you expect: like numbers. - - diff --git a/content/constants/bool.go b/content/constants/bool.go deleted file mode 100644 index 7578d4ac..00000000 --- a/content/constants/bool.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - type MyBool bool - const True = true - const TypedTrue bool = true - var mb MyBool - mb = true // OK - mb = True // OK - mb = TypedTrue // Bad - fmt.Println(mb) - // STOP OMIT -} diff --git a/content/constants/complex1.go b/content/constants/complex1.go deleted file mode 100644 index ea459218..00000000 --- a/content/constants/complex1.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - type MyComplex128 complex128 - const I = (0.0 + 1.0i) - const TypedI complex128 = (0.0 + 1.0i) - var mc MyComplex128 - mc = (0.0 + 1.0i) // OK - mc = I // OK - mc = TypedI // Bad - fmt.Println(mc) - // STOP OMIT -} diff --git a/content/constants/complex2.go b/content/constants/complex2.go deleted file mode 100644 index 5dd4f730..00000000 --- a/content/constants/complex2.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - const Two = 2.0 + 0i - // START OMIT - s := Two - fmt.Printf("%T: %v\n", s, s) - // STOP OMIT -} diff --git a/content/constants/complex3.go b/content/constants/complex3.go deleted file mode 100644 index 95e9e418..00000000 --- a/content/constants/complex3.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - const Two = 2.0 + 0i - // START OMIT - var f float64 - var g float64 = Two - f = Two - fmt.Println(f, "and", g) - // STOP OMIT -} diff --git a/content/constants/default1.go b/content/constants/default1.go deleted file mode 100644 index 0f7909cd..00000000 --- a/content/constants/default1.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - fmt.Printf("%s", "Hello, 世界") - // STOP OMIT -} diff --git a/content/constants/default2.go b/content/constants/default2.go deleted file mode 100644 index 2183cf03..00000000 --- a/content/constants/default2.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -const hello = "Hello, 世界" - -func main() { - // START OMIT - fmt.Printf("%T: %v\n", "Hello, 世界", "Hello, 世界") - fmt.Printf("%T: %v\n", hello, hello) - // STOP OMIT -} diff --git a/content/constants/default3.go b/content/constants/default3.go deleted file mode 100644 index 246ae453..00000000 --- a/content/constants/default3.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -type MyString string - -const myStringHello MyString = "Hello, 世界" - -func main() { - // START OMIT - fmt.Printf("%T: %v\n", myStringHello, myStringHello) - // STOP OMIT -} diff --git a/content/constants/exercise1.go b/content/constants/exercise1.go deleted file mode 100644 index 881d1d04..00000000 --- a/content/constants/exercise1.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - const MaxUint uint = -1 // Error: negative value - // STOP OMIT -} diff --git a/content/constants/exercise2.go b/content/constants/exercise2.go deleted file mode 100644 index db7d8c17..00000000 --- a/content/constants/exercise2.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - const MaxUint uint = uint(-1) // Error: negative value - // STOP OMIT -} diff --git a/content/constants/exercise3.go b/content/constants/exercise3.go deleted file mode 100644 index dbe75954..00000000 --- a/content/constants/exercise3.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - var u uint - var v = -1 - u = uint(v) - // STOP OMIT - _ = u -} diff --git a/content/constants/exercise4.go b/content/constants/exercise4.go deleted file mode 100644 index 3acc664a..00000000 --- a/content/constants/exercise4.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - var u uint - const v = -1 - u = uint(v) // Error: negative value - // STOP OMIT - _ = u -} diff --git a/content/constants/exercise5.go b/content/constants/exercise5.go deleted file mode 100644 index 2fca330d..00000000 --- a/content/constants/exercise5.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - const MaxUint uint = ^0 // Error: overflow - // STOP OMIT -} diff --git a/content/constants/exercise6.go b/content/constants/exercise6.go deleted file mode 100644 index ce6478c7..00000000 --- a/content/constants/exercise6.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - const MaxUint = ^uint(0) - fmt.Printf("%x\n", MaxUint) - // STOP OMIT -} diff --git a/content/constants/float1.go b/content/constants/float1.go deleted file mode 100644 index 96b3bb92..00000000 --- a/content/constants/float1.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - type MyFloat64 float64 - const Zero = 0.0 - const TypedZero float64 = 0.0 - var mf MyFloat64 - mf = 0.0 // OK - mf = Zero // OK - mf = TypedZero // Bad - fmt.Println(mf) - // STOP OMIT -} diff --git a/content/constants/float2.go b/content/constants/float2.go deleted file mode 100644 index f43feb63..00000000 --- a/content/constants/float2.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - const Zero = 0.0 - const TypedZero float64 = 0.0 - // START OMIT - var f32 float32 - f32 = 0.0 - f32 = Zero // OK: Zero is untyped - f32 = TypedZero // Bad: TypedZero is float64 not float32. - fmt.Println(f32) - // STOP OMIT -} diff --git a/content/constants/float3.go b/content/constants/float3.go deleted file mode 100644 index d862acce..00000000 --- a/content/constants/float3.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - const Huge = 1e1000 - // START OMIT - fmt.Println(Huge) - // STOP OMIT -} diff --git a/content/constants/float4.go b/content/constants/float4.go deleted file mode 100644 index 206d215c..00000000 --- a/content/constants/float4.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - const Huge = 1e1000 - // START OMIT - fmt.Println(Huge / 1e999) - // STOP OMIT -} diff --git a/content/constants/float5.go b/content/constants/float5.go deleted file mode 100644 index 2d628721..00000000 --- a/content/constants/float5.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "math" -) - -func main() { - // START OMIT - pi := math.Pi - fmt.Println(pi) - // STOP OMIT -} diff --git a/content/constants/int1.go b/content/constants/int1.go deleted file mode 100644 index 913ad781..00000000 --- a/content/constants/int1.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - type MyInt int - const Three = 3 - const TypedThree int = 3 - var mi MyInt - mi = 3 // OK - mi = Three // OK - mi = TypedThree // Bad - fmt.Println(mi) - // STOP OMIT -} diff --git a/content/constants/int2.go b/content/constants/int2.go deleted file mode 100644 index 0e974ef2..00000000 --- a/content/constants/int2.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - var i8 int8 = 128 // Error: too large. - // STOP OMIT - _ = i8 -} diff --git a/content/constants/int3.go b/content/constants/int3.go deleted file mode 100644 index c2cec6ed..00000000 --- a/content/constants/int3.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - var u8 uint8 = -1 // Error: negative value. - // STOP OMIT - _ = u8 -} diff --git a/content/constants/int4.go b/content/constants/int4.go deleted file mode 100644 index caff695a..00000000 --- a/content/constants/int4.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build OMIT - -package main - -func main() { - // START OMIT - type Char byte - var c Char = '世' // Error: '世' has value 0x4e16, too large. - // STOP OMIT - _ = c -} diff --git a/content/constants/numbers1.go b/content/constants/numbers1.go deleted file mode 100644 index ea35a514..00000000 --- a/content/constants/numbers1.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - var f float32 = 1 - var i int = 1.000 - var u uint32 = 1e3 - 99.0*10.0 - 9 - var c float64 = '\x01' - var p uintptr = '\u0001' - var r complex64 = 'b' - 'a' - var b byte = 1.0 + 3i - 3.0i - - fmt.Println(f, i, u, c, p, r, b) - // STOP OMIT -} diff --git a/content/constants/numbers2.go b/content/constants/numbers2.go deleted file mode 100644 index 33b7de02..00000000 --- a/content/constants/numbers2.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - var f = 'a' * 1.5 - fmt.Println(f) - // STOP OMIT -} diff --git a/content/constants/string1.go b/content/constants/string1.go deleted file mode 100644 index 3b83851b..00000000 --- a/content/constants/string1.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -const typedHello string = "Hello, 世界" - -func main() { - // START OMIT - var s string - s = typedHello - fmt.Println(s) - // STOP OMIT -} diff --git a/content/constants/string2.go b/content/constants/string2.go deleted file mode 100644 index 362b4841..00000000 --- a/content/constants/string2.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -const typedHello string = "Hello, 世界" - -func main() { - // START OMIT - type MyString string - var m MyString - m = typedHello // Type error - fmt.Println(m) - // STOP OMIT -} diff --git a/content/constants/string3.go b/content/constants/string3.go deleted file mode 100644 index 72de35f5..00000000 --- a/content/constants/string3.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -const typedHello string = "Hello, 世界" - -func main() { - type MyString string - var m MyString - // START OMIT - const myStringHello MyString = "Hello, 世界" - m = myStringHello // OK - fmt.Println(m) - // STOP OMIT -} diff --git a/content/constants/string4.go b/content/constants/string4.go deleted file mode 100644 index 4c35f297..00000000 --- a/content/constants/string4.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -const typedHello string = "Hello, 世界" - -func main() { - type MyString string - var m MyString - // START OMIT - m = MyString(typedHello) - fmt.Println(m) - // STOP OMIT -} diff --git a/content/constants/syntax.go b/content/constants/syntax.go deleted file mode 100644 index 16e7e641..00000000 --- a/content/constants/syntax.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - fmt.Printf("%T %v\n", 0, 0) - fmt.Printf("%T %v\n", 0.0, 0.0) - fmt.Printf("%T %v\n", 'x', 'x') - fmt.Printf("%T %v\n", 0i, 0i) - // STOP OMIT -} diff --git a/content/context.article b/content/context.article deleted file mode 100644 index 1d121f3f..00000000 --- a/content/context.article +++ /dev/null @@ -1,224 +0,0 @@ -Go Concurrency Patterns: Context -29 Jul 2014 -Tags: concurrency, cancelation, cancellation, context - -Sameer Ajmani - -* Introduction - -In Go servers, each incoming request is handled in its own goroutine. -Request handlers often start additional goroutines to access backends such as -databases and RPC services. -The set of goroutines working on a request typically needs access to -request-specific values such as the identity of the end user, authorization -tokens, and the request's deadline. -When a request is canceled or times out, all the goroutines working on that -request should exit quickly so the system can reclaim any resources they are -using. - -At Google, we developed a `context` package that makes it easy to pass -request-scoped values, cancelation signals, and deadlines across API boundaries -to all the goroutines involved in handling a request. -The package is publicly available as -[[http://godoc.org/golang.org/x/net/context][golang.org/x/net/context]]. -This article describes how to use the package and provides a complete working -example. - -* Context - -The core of the `context` package is the `Context` type: - -.code context/interface.go /A Context/,/^}/ - -(This description is condensed; the -[[http://godoc.org/golang.org/x/net/context][godoc]] is authoritative.) - -The `Done` method returns a channel that acts as a cancelation signal to -functions running on behalf of the `Context`: when the channel is closed, the -functions should abandon their work and return. -The `Err` method returns an error indicating why the `Context` was canceled. -The [[/pipelines][Pipelines and Cancelation]] article discusses the `Done` -channel idiom in more detail. - -A `Context` does _not_ have a `Cancel` method for the same reason the `Done` -channel is receive-only: the function receiving a cancelation signal is usually -not the one that sends the signal. -In particular, when a parent operation starts goroutines for sub-operations, -those sub-operations should not be able to cancel the parent. -Instead, the `WithCancel` function (described below) provides a way to cancel a -new `Context` value. - -A `Context` is safe for simultaneous use by multiple goroutines. -Code can pass a single `Context` to any number of goroutines and cancel that -`Context` to signal all of them. - -The `Deadline` method allows functions to determine whether they should start -work at all; if too little time is left, it may not be worthwhile. -Code may also use a deadline to set timeouts for I/O operations. - -`Value` allows a `Context` to carry request-scoped data. -That data must be safe for simultaneous use by multiple goroutines. - -** Derived contexts - -The `context` package provides functions to _derive_ new `Context` values from -existing ones. -These values form a tree: when a `Context` is canceled, all `Contexts` derived -from it are also canceled. - -`Background` is the root of any `Context` tree; it is never canceled: - -.code context/interface.go /Background returns/,/func Background/ - -`WithCancel` and `WithTimeout` return derived `Context` values that can be -canceled sooner than the parent `Context`. -The `Context` associated with an incoming request is typically canceled when the -request handler returns. -`WithCancel` is also useful for canceling redundant requests when using multiple -replicas. -`WithTimeout` is useful for setting a deadline on requests to backend servers: - -.code context/interface.go /WithCancel/,/func WithTimeout/ - -`WithValue` provides a way to associate request-scoped values with a `Context`: - -.code context/interface.go /WithValue/,/func WithValue/ - -The best way to see how to use the `context` package is through a worked -example. - -* Example: Google Web Search - -Our example is an HTTP server that handles URLs like -`/search?q=golang&timeout=1s` by forwarding the query "golang" to the -[[https://developers.google.com/web-search/docs/][Google Web Search API]] and -rendering the results. -The `timeout` parameter tells the server to cancel the request after that -duration elapses. - -The code is split across three packages: - -- [[context/server/server.go][server]] provides the `main` function and the handler for `/search`. -- [[context/userip/userip.go][userip]] provides functions for extracting a user IP address from a request and associating it with a `Context`. -- [[context/google/google.go][google]] provides the `Search` function for sending a query to Google. - -** The server program - -The [[context/server/server.go][server]] program handles requests like -`/search?q=golang` by serving the first few Google search results for `golang`. -It registers `handleSearch` to handle the `/search` endpoint. -The handler creates an initial `Context` called `ctx` and arranges for it to be -canceled when the handler returns. -If the request includes the `timeout` URL parameter, the `Context` is canceled -automatically when the timeout elapses: - -.code context/server/server.go /func handleSearch/,/defer cancel/ - -The handler extracts the query from the request and extracts the client's IP -address by calling on the `userip` package. -The client's IP address is needed for backend requests, so `handleSearch` -attaches it to `ctx`: - -.code context/server/server.go /Check the search query/,/userip.NewContext/ - -The handler calls `google.Search` with `ctx` and the `query`: - -.code context/server/server.go /Run the Google search/,/elapsed/ - -If the search succeeds, the handler renders the results: - -.code context/server/server.go /resultsTemplate/,/}$/ - -** Package userip - -The [[context/userip/userip.go][userip]] package provides functions for -extracting a user IP address from a request and associating it with a `Context`. -A `Context` provides a key-value mapping, where the keys and values are both of -type `interface{}`. -Key types must support equality, and values must be safe for simultaneous use by -multiple goroutines. -Packages like `userip` hide the details of this mapping and provide -strongly-typed access to a specific `Context` value. - -To avoid key collisions, `userip` defines an unexported type `key` and uses -a value of this type as the context key: - -.code context/userip/userip.go /The key type/,/const userIPKey/ - -`FromRequest` extracts a `userIP` value from an `http.Request`: - -.code context/userip/userip.go /func FromRequest/,/}/ - -`NewContext` returns a new `Context` that carries a provided `userIP` value: - -.code context/userip/userip.go /func NewContext/,/}/ - -`FromContext` extracts a `userIP` from a `Context`: - -.code context/userip/userip.go /func FromContext/,/}/ - -** Package google - -The [[context/google/google.go][google.Search]] function makes an HTTP request -to the [[https://developers.google.com/web-search/docs/][Google Web Search API]] -and parses the JSON-encoded result. -It accepts a `Context` parameter `ctx` and returns immediately if `ctx.Done` is -closed while the request is in flight. - -The Google Web Search API request includes the search query and the user IP as -query parameters: - -.code context/google/google.go /func Search/,/q.Encode/ - -`Search` uses a helper function, `httpDo`, to issue the HTTP request and cancel -it if `ctx.Done` is closed while the request or response is being processed. -`Search` passes a closure to `httpDo` handle the HTTP response: - -.code context/google/google.go /var results/,/return results/ - -The `httpDo` function runs the HTTP request and processes its response in a new -goroutine. -It cancels the request if `ctx.Done` is closed before the goroutine exits: - -.code context/google/google.go /func httpDo/,/^}/ - -* Adapting code for Contexts - -Many server frameworks provide packages and types for carrying request-scoped -values. -We can define new implementations of the `Context` interface to bridge between -code using existing frameworks and code that expects a `Context` parameter. - -For example, Gorilla's -[[http://www.gorillatoolkit.org/pkg/context][github.com/gorilla/context]] -package allows handlers to associate data with incoming requests by providing a -mapping from HTTP requests to key-value pairs. -In [[context/gorilla/gorilla.go][gorilla.go]], we provide a `Context` -implementation whose `Value` method returns the values associated with a -specific HTTP request in the Gorilla package. - -Other packages have provided cancelation support similar to `Context`. -For example, [[http://godoc.org/gopkg.in/tomb.v2][Tomb]] provides a `Kill` -method that signals cancelation by closing a `Dying` channel. -`Tomb` also provides methods to wait for those goroutines to exit, similar to -`sync.WaitGroup`. -In [[context/tomb/tomb.go][tomb.go]], we provide a `Context` implementation that -is canceled when either its parent `Context` is canceled or a provided `Tomb` is -killed. - -* Conclusion - -At Google, we require that Go programmers pass a `Context` parameter as the -first argument to every function on the call path between incoming and outgoing -requests. -This allows Go code developed by many different teams to interoperate well. -It provides simple control over timeouts and cancelation and ensures that -critical values like security credentials transit Go programs properly. - -Server frameworks that want to build on `Context` should provide implementations -of `Context` to bridge between their packages and those that expect a `Context` -parameter. -Their client libraries would then accept a `Context` from the calling code. -By establishing a common interface for request-scoped data and cancelation, -`Context` makes it easier for package developers to share code for creating -scalable services. diff --git a/content/context/google/google.go b/content/context/google/google.go deleted file mode 100644 index 73778a3d..00000000 --- a/content/context/google/google.go +++ /dev/null @@ -1,88 +0,0 @@ -// Package google provides a function to do Google searches using the Google Web -// Search API. See https://developers.google.com/web-search/docs/ -package google - -import ( - "encoding/json" - "net/http" - - "golang.org/x/blog/content/context/userip" - "golang.org/x/net/context" -) - -// Results is an ordered list of search results. -type Results []Result - -// A Result contains the title and URL of a search result. -type Result struct { - Title, URL string -} - -// Search sends query to Google search and returns the results. -func Search(ctx context.Context, query string) (Results, error) { - // Prepare the Google Search API request. - req, err := http.NewRequest("GET", "https://ajax.googleapis.com/ajax/services/search/web?v=1.0", nil) - if err != nil { - return nil, err - } - q := req.URL.Query() - q.Set("q", query) - - // If ctx is carrying the user IP address, forward it to the server. - // Google APIs use the user IP to distinguish server-initiated requests - // from end-user requests. - if userIP, ok := userip.FromContext(ctx); ok { - q.Set("userip", userIP.String()) - } - req.URL.RawQuery = q.Encode() - - // Issue the HTTP request and handle the response. The httpDo function - // cancels the request if ctx.Done is closed. - var results Results - err = httpDo(ctx, req, func(resp *http.Response, err error) error { - if err != nil { - return err - } - defer resp.Body.Close() - - // Parse the JSON search result. - // https://developers.google.com/web-search/docs/#fonje - var data struct { - ResponseData struct { - Results []struct { - TitleNoFormatting string - URL string - } - } - } - if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { - return err - } - for _, res := range data.ResponseData.Results { - results = append(results, Result{Title: res.TitleNoFormatting, URL: res.URL}) - } - return nil - }) - // httpDo waits for the closure we provided to return, so it's safe to - // read results here. - return results, err -} - -// httpDo issues the HTTP request and calls f with the response. If ctx.Done is -// closed while the request or f is running, httpDo cancels the request, waits -// for f to exit, and returns ctx.Err. Otherwise, httpDo returns f's error. -func httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error { - // Run the HTTP request in a goroutine and pass the response to f. - tr := &http.Transport{} - client := &http.Client{Transport: tr} - c := make(chan error, 1) - go func() { c <- f(client.Do(req)) }() - select { - case <-ctx.Done(): - tr.CancelRequest(req) - <-c // Wait for f to return. - return ctx.Err() - case err := <-c: - return err - } -} diff --git a/content/context/gorilla/gorilla.go b/content/context/gorilla/gorilla.go deleted file mode 100644 index 4698dce4..00000000 --- a/content/context/gorilla/gorilla.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build OMIT - -// Package gorilla provides a go.net/context.Context implementation whose Value -// method returns the values associated with a specific HTTP request in the -// github.com/gorilla/context package. -package gorilla - -import ( - "net/http" - - gcontext "github.com/gorilla/context" - "golang.org/x/net/context" -) - -// NewContext returns a Context whose Value method returns values associated -// with req using the Gorilla context package: -// http://www.gorillatoolkit.org/pkg/context -func NewContext(parent context.Context, req *http.Request) context.Context { - return &wrapper{parent, req} -} - -type wrapper struct { - context.Context - req *http.Request -} - -type key int - -const reqKey key = 0 - -// Value returns Gorilla's context package's value for this Context's request -// and key. It delegates to the parent Context if there is no such value. -func (ctx *wrapper) Value(key interface{}) interface{} { - if key == reqKey { - return ctx.req - } - if val, ok := gcontext.GetOk(ctx.req, key); ok { - return val - } - return ctx.Context.Value(key) -} - -// HTTPRequest returns the *http.Request associated with ctx using NewContext, -// if any. -func HTTPRequest(ctx context.Context) (*http.Request, bool) { - // We cannot use ctx.(*wrapper).req to get the request because ctx may - // be a Context derived from a *wrapper. Instead, we use Value to - // access the request if it is anywhere up the Context tree. - req, ok := ctx.Value(reqKey).(*http.Request) - return req, ok -} diff --git a/content/context/interface.go b/content/context/interface.go deleted file mode 100644 index 838c7da8..00000000 --- a/content/context/interface.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build OMIT - -package context - -import "time" - -// A Context carries a deadline, cancelation signal, and request-scoped values -// across API boundaries. Its methods are safe for simultaneous use by multiple -// goroutines. -type Context interface { - // Done returns a channel that is closed when this Context is canceled - // or times out. - Done() <-chan struct{} - - // Err indicates why this context was canceled, after the Done channel - // is closed. - Err() error - - // Deadline returns the time when this Context will be canceled, if any. - Deadline() (deadline time.Time, ok bool) - - // Value returns the value associated with key or nil if none. - Value(key interface{}) interface{} -} - -// WithCancel returns a copy of parent whose Done channel is closed as soon as -// parent.Done is closed or cancel is called. -func WithCancel(parent Context) (ctx Context, cancel CancelFunc) - -// A CancelFunc cancels a Context. -type CancelFunc func() - -// WithTimeout returns a copy of parent whose Done channel is closed as soon as -// parent.Done is closed, cancel is called, or timeout elapses. The new -// Context's Deadline is the sooner of now+timeout and the parent's deadline, if -// any. If the timer is still running, the cancel function releases its -// resources. -func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) - -// WithValue returns a copy of parent whose Value method returns val for key. -func WithValue(parent Context, key interface{}, val interface{}) Context - -// Background returns an empty Context. It is never canceled, has no deadline, -// and has no values. Background is typically used in main, init, and tests, -// and as the top-level Context for incoming requests. -func Background() Context diff --git a/content/context/server/server.go b/content/context/server/server.go deleted file mode 100644 index 25043703..00000000 --- a/content/context/server/server.go +++ /dev/null @@ -1,98 +0,0 @@ -// The server program issues Google search requests and demonstrates the use of -// the go.net Context API. It serves on port 8080. -// -// The /search endpoint accepts these query params: -// q=the Google search query -// timeout=a timeout for the request, in time.Duration format -// -// For example, http://localhost:8080/search?q=golang&timeout=1s serves the -// first few Google search results for "golang" or a "deadline exceeded" error -// if the timeout expires. -package main - -import ( - "html/template" - "log" - "net/http" - "time" - - "golang.org/x/blog/content/context/google" - "golang.org/x/blog/content/context/userip" - "golang.org/x/net/context" -) - -func main() { - http.HandleFunc("/search", handleSearch) - log.Fatal(http.ListenAndServe(":8080", nil)) -} - -// handleSearch handles URLs like /search?q=golang&timeout=1s by forwarding the -// query to google.Search. If the query param includes timeout, the search is -// canceled after that duration elapses. -func handleSearch(w http.ResponseWriter, req *http.Request) { - // ctx is the Context for this handler. Calling cancel closes the - // ctx.Done channel, which is the cancellation signal for requests - // started by this handler. - var ( - ctx context.Context - cancel context.CancelFunc - ) - timeout, err := time.ParseDuration(req.FormValue("timeout")) - if err == nil { - // The request has a timeout, so create a context that is - // canceled automatically when the timeout expires. - ctx, cancel = context.WithTimeout(context.Background(), timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() // Cancel ctx as soon as handleSearch returns. - - // Check the search query. - query := req.FormValue("q") - if query == "" { - http.Error(w, "no query", http.StatusBadRequest) - return - } - - // Store the user IP in ctx for use by code in other packages. - userIP, err := userip.FromRequest(req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - ctx = userip.NewContext(ctx, userIP) - - // Run the Google search and print the results. - start := time.Now() - results, err := google.Search(ctx, query) - elapsed := time.Since(start) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err := resultsTemplate.Execute(w, struct { - Results google.Results - Timeout, Elapsed time.Duration - }{ - Results: results, - Timeout: timeout, - Elapsed: elapsed, - }); err != nil { - log.Print(err) - return - } -} - -var resultsTemplate = template.Must(template.New("results").Parse(` - - - -
    - {{range .Results}} -
  1. {{.Title}} - {{.URL}}
    2. - {{end}} -
-

{{len .Results}} results in {{.Elapsed}}; timeout {{.Timeout}}

- - -`)) diff --git a/content/context/tomb/tomb.go b/content/context/tomb/tomb.go deleted file mode 100644 index da94dc9a..00000000 --- a/content/context/tomb/tomb.go +++ /dev/null @@ -1,24 +0,0 @@ -// +build OMIT - -// Package tomb provides a Context implementation that is canceled when either -// its parent Context is canceled or a provided Tomb is killed. -package tomb - -import ( - "golang.org/x/net/context" - tomb "gopkg.in/tomb.v2" -) - -// NewContext returns a Context that is canceled either when parent is canceled -// or when t is Killed. -func NewContext(parent context.Context, t *tomb.Tomb) context.Context { - ctx, cancel := context.WithCancel(parent) - go func() { - select { - case <-t.Dying(): - cancel() - case <-ctx.Done(): - } - }() - return ctx -} diff --git a/content/context/userip/userip.go b/content/context/userip/userip.go deleted file mode 100644 index 4f316b56..00000000 --- a/content/context/userip/userip.go +++ /dev/null @@ -1,47 +0,0 @@ -// Package userip provides functions for extracting a user IP address from a -// request and associating it with a Context. -package userip - -import ( - "fmt" - "net" - "net/http" - - "golang.org/x/net/context" -) - -// FromRequest extracts the user IP address from req, if present. -func FromRequest(req *http.Request) (net.IP, error) { - ip, _, err := net.SplitHostPort(req.RemoteAddr) - if err != nil { - return nil, fmt.Errorf("userip: %q is not IP:port", req.RemoteAddr) - } - - userIP := net.ParseIP(ip) - if userIP == nil { - return nil, fmt.Errorf("userip: %q is not IP:port", req.RemoteAddr) - } - return userIP, nil -} - -// The key type is unexported to prevent collisions with context keys defined in -// other packages. -type key int - -// userIPkey is the context key for the user IP address. Its value of zero is -// arbitrary. If this package defined other context keys, they would have -// different integer values. -const userIPKey key = 0 - -// NewContext returns a new Context carrying userIP. -func NewContext(ctx context.Context, userIP net.IP) context.Context { - return context.WithValue(ctx, userIPKey, userIP) -} - -// FromContext extracts the user IP address from ctx, if present. -func FromContext(ctx context.Context) (net.IP, bool) { - // ctx.Value returns nil if ctx has no value for the key; - // the net.IP type assertion returns ok=false for nil. - userIP, ok := ctx.Value(userIPKey).(net.IP) - return userIP, ok -} diff --git a/content/cover.article b/content/cover.article deleted file mode 100644 index 4e6bcdba..00000000 --- a/content/cover.article +++ /dev/null @@ -1,299 +0,0 @@ -The cover story -2 Dec 2013 -Tags: tools, coverage, testing - -Rob Pike - -* Introduction - -From the beginning of the project, Go was designed with tools in mind. -Those tools include some of the most iconic pieces of Go technology such as -the documentation presentation tool -[[http://golang.org/cmd/godoc][godoc]], -the code formatting tool -[[http://golang.org/cmd/gofmt][gofmt]], -and the API rewriter -[[http://golang.org/cmd/fix][gofix]]. -Perhaps most important of all is the -[[http://golang.org/cmd/go][`go` command]], -the program that automatically installs, builds, and tests Go programs -using nothing more than the source code as the build specification. - -The release of Go 1.2 introduces a new tool for test coverage that takes an -unusual approach to the way it generates coverage statistics, an approach -that builds on the technology laid down by godoc and friends. - -* Support for tools - -First, some background: What does it mean for a -[[http://talks.golang.org/2012/splash.article#TOC_17.][language to support good tooling]]? -It means that the language makes it easy to write good tools and that its ecosystem -supports the construction of tools of all flavors. - -There are a number of properties of Go that make it suitable for tooling. -For starters, Go has a regular syntax that is easy to parse. -The grammar aims to be free of special cases that require complex machinery to analyze. - -Where possible, Go uses lexical and syntactic constructs to make semantic properties -easy to understand. -Examples include the use of upper-case letters to define exported names -and the radically simplified scoping rules compared to other languages in the C tradition. - -Finally, the standard library comes with production-quality packages to lex and parse Go source code. -They also include, more unusually, a production-quality package to pretty-print Go syntax trees. - -These packages in combination form the core of the gofmt tool, but the pretty-printer is worth singling out. -Because it can take an arbitrary Go syntax tree and output standard-format, human-readable, correct -code, it creates the possibility to build tools that transform the parse tree and output modified but -correct and easy-to-read code. - -One example is the gofix tool, which automates the -rewriting of code to use new language features or updated libraries. -Gofix let us make fundamental changes to the language and libraries in the -[[http://blog.golang.org/the-path-to-go-1][run-up to Go 1.0]], -with the confidence that users could just run the tool to update their source to the newest version. - -Inside Google, we have used gofix to make sweeping changes in a huge code repository that would be almost -unthinkable in the other languages we use. -There's no need any more to support multiple versions of some API; we can use gofix to update -the entire company in one operation. - -It's not just these big tools that these packages enable, of course. -They also make it easy to write more modest programs such as IDE plugins, for instance. -All these items build on each other, making the Go environment -more productive by automating many tasks. - -* Test coverage - -Test coverage is a term that describes how much of a package's code is exercised by running the package's tests. -If executing the test suite causes 80% of the package's source statements to be run, we say that the test coverage is 80%. - -The program that provides test coverage in Go 1.2 is the latest to exploit the tooling support in the Go ecosystem. - -The usual way to compute test coverage is to instrument the binary. -For instance, the GNU [[http://gcc.gnu.org/onlinedocs/gcc/Gcov.html][gcov]] program sets breakpoints at branches -executed by the binary. -As each branch executes, the breakpoint is cleared and the target statements of the branch are marked as 'covered'. - -This approach is successful and widely used. An early test coverage tool for Go even worked the same way. -But it has problems. -It is difficult to implement, as analysis of the execution of binaries is challenging. -It also requires a reliable way of tying the execution trace back to the source code, which can also be difficult, -as any user of a source-level debugger can attest. -Problems there include inaccurate debugging information and issues such as in-lined functions complicating -the analysis. -Most important, this approach is very non-portable. -It needs to be done afresh for every architecture, and to some extent for every -operating system since debugging support varies greatly from system to system. - -It does work, though, and for instance if you are a user of gccgo, the gcov tool can give you test coverage -information. -However If you're a user of gc, the more commonly used Go compiler suite, until Go 1.2 you were out of luck. - -* Test coverage for Go - -For the new test coverage tool for Go, we took a different approach that avoids dynamic debugging. -The idea is simple: Rewrite the package's source code before compilation to add instrumentation, -compile and run the modified source, and dump the statistics. -The rewriting is easy to arrange because the `go` command controls the flow -from source to test to execution. - -Here's an example. Say we have a simple, one-file package like this: - -.code cover/pkg.go - -and this test: - -.code cover/pkg_test.go - -To get the test coverage for the package, -we run the test with coverage enabled by providing the `-cover` flag to `go` `test`: - - % go test -cover - PASS - coverage: 42.9% of statements - ok size 0.026s - % - -Notice that the coverage is 42.9%, which isn't very good. -Before we ask how to raise that number, let's see how that was computed. - -When test coverage is enabled, `go` `test` runs the "cover" tool, a separate program included -with the distribution, to rewrite the source code before compilation. Here's what the rewritten -`Size` function looks like: - -.code cover/pkg.cover /func/,/^}/ - -Each executable section of the program is annotated with an assignment statement that, -when executed, records that that section ran. -The counter is tied to the original source position of the statements it counts -through a second read-only data structure that is also generated by the cover tool. -When the test run completes, the counters are collected and the percentage is computed -by seeing how many were set. - -Although that annotating assignment might look expensive, it compiles to a single "move" instruction. -Its run-time overhead is therefore modest, adding only about 3% when running a typical (more realistic) test. -That makes it reasonable to include test coverage as part of the standard development pipeline. - -* Viewing the results - -The test coverage for our example was poor. -To discover why, we ask `go` `test` to write a "coverage profile" for us, a file that holds -the collected statistics so we can study them in more detail. -That's easy to do: use the `-coverprofile` flag to specify a file for the output: - - % go test -coverprofile=coverage.out - PASS - coverage: 42.9% of statements - ok size 0.030s - % - -(The `-coverprofile` flag automatically sets `-cover` to enable coverage analysis.) -The test runs just as before, but the results are saved in a file. -To study them, we run the test coverage tool ourselves, without `go` `test`. -As a start, we can ask for the coverage to be broken down by function, -although that's not going to illuminate much in this case since there's -only one function: - - % go tool cover -func=coverage.out - size.go: Size 42.9% - total: (statements) 42.9% - % - -A much more interesting way to see the data is to get an HTML presentation -of the source code decorated with coverage information. -This display is invoked by the `-html` flag: - - $ go tool cover -html=coverage.out - -When this command is run, a browser window pops up, showing the covered (green), -uncovered (red), and uninstrumented (grey) source. -Here's a screen dump: - -.image cover/set.png - -With this presentation, it's obvious what's wrong: we neglected to test several -of the cases! -And we can see exactly which ones they are, which makes it easy to -improve our test coverage. - -* Heat maps - -A big advantage of this source-level approach to test coverage is that it's -easy to instrument the code in different ways. -For instance, we can ask not only whether a statement has been executed, -but how many times. - -The `go` `test` command accepts a `-covermode` flag to set the coverage mode -to one of three settings: - -- set: did each statement run? -- count: how many times did each statement run? -- atomic: like count, but counts precisely in parallel programs - -The default is 'set', which we've already seen. -The `atomic` setting is needed only when accurate counts are required -when running parallel algorithms. It uses atomic operations from the -[[http://golang.org/pkg/sync/atomic/][sync/atomic]] package, -which can be quite expensive. -For most purposes, though, the `count` mode works fine and, like -the default `set` mode, is very cheap. - -Let's try counting statement execution for a standard package, the `fmt` formatting package. -We run the test and write out a coverage profile so we can present the information -nicely afterwards. - - % go test -covermode=count -coverprofile=count.out fmt - ok fmt 0.056s coverage: 91.7% of statements - % - -That's a much better test coverage ratio than for our previous example. -(The coverage ratio is not affected by the coverage mode.) -We can display the function breakdown: - - % go tool cover -func=count.out - fmt/format.go: init 100.0% - fmt/format.go: clearflags 100.0% - fmt/format.go: init 100.0% - fmt/format.go: computePadding 84.6% - fmt/format.go: writePadding 100.0% - fmt/format.go: pad 100.0% - ... - fmt/scan.go: advance 96.2% - fmt/scan.go: doScanf 96.8% - total: (statements) 91.7% - -The big payoff happens in the HTML output: - - % go tool cover -html=count.out - -Here's what the `pad` function looks like in that presentation: - -.image cover/count.png - -Notice how the intensity of the green changes. Brighter-green -statements have higher execution counts; less saturated greens -represent lower execution counts. -You can even hover the mouse over the statements to see the -actual counts pop up in a tool tip. -At the time of writing, the counts come out like this -(we've moved the counts from the tool tips to beginning-of-line -markers to make them easier to show): - - 2933 if !f.widPresent || f.wid == 0 { - 2985 f.buf.Write(b) - 2985 return - 2985 } - 56 padding, left, right := f.computePadding(len(b)) - 56 if left > 0 { - 37 f.writePadding(left, padding) - 37 } - 56 f.buf.Write(b) - 56 if right > 0 { - 13 f.writePadding(right, padding) - 13 } - -That's a lot of information about the execution of the function, -information that might be useful in profiling. - -* Basic blocks - -You might have noticed that the counts in the previous example -were not what you expected on the lines with closing braces. -That's because, as always, test coverage is an inexact science. - -What's going on here is worth explaining, though. We'd like the -coverage annotations to be demarcated by branches in the program, -the way they are when the binary is instrumented in the traditional -method. -It's hard to do that by rewriting the source, though, since -the branches don't appear explicitly in the source. - -What the coverage annotation does is instrument blocks, which -are typically bounded by brace brackets. -Getting this right in general is very hard. -A consequence of the algorithm used is that the closing -brace looks like it belongs to the block it closes, while the -opening brace looks like it belongs outside the block. -A more interesting consequence is that in an expression like - - f() && g() - -there is no attempt to separately instrument the calls to `f` and `g`, Regardless of -the facts it will always look like they both ran the same -number of times, the number of times `f` ran. - -To be fair, even `gcov` has trouble here. That tool gets the -instrumentation right but the presentation is line-based and -can therefore miss some nuances. - -* The big picture - -That's the story about test coverage in Go 1.2. -A new tool with an interesting implementation enables not only -test coverage statistics, but easy-to-interpret presentations -of them and even the possibility to extract profiling information. - -Testing is an important part of software development and test -coverage a simple way to add discipline to your testing strategy. -Go forth, test, and cover. diff --git a/content/cover/count.png b/content/cover/count.png deleted file mode 100644 index 2c669565..00000000 Binary files a/content/cover/count.png and /dev/null differ diff --git a/content/cover/pkg.cover b/content/cover/pkg.cover deleted file mode 100644 index 03a8cbca..00000000 --- a/content/cover/pkg.cover +++ /dev/null @@ -1,49 +0,0 @@ -package main - -func Size(a int) string { - GoCover.Count[0] = 1 - switch { - case a < 0: - GoCover.Count[2] = 1 - return "negative" - case a == 0: - GoCover.Count[3] = 1 - return "zero" - case a < 10: - GoCover.Count[4] = 1 - return "small" - case a < 100: - GoCover.Count[5] = 1 - return "big" - case a < 1000: - GoCover.Count[6] = 1 - return "huge" - } - GoCover.Count[1] = 1 - return "enormous" -} - -var GoCover = struct { - Count [7]uint32 - Pos [3 * 7]uint32 - NumStmt [7]uint16 -} { - Pos: [3 * 7]uint32{ - 3, 4, 0x90019, // [0] - 16, 16, 0x130002, // [1] - 5, 6, 0x140002, // [2] - 7, 8, 0x100002, // [3] - 9, 10, 0x110002, // [4] - 11, 12, 0xf0002, // [5] - 13, 14, 0x100002, // [6] - }, - NumStmt: [7]uint16{ - 1, // 0 - 1, // 1 - 1, // 2 - 1, // 3 - 1, // 4 - 1, // 5 - 1, // 6 - }, -} diff --git a/content/cover/pkg.go b/content/cover/pkg.go deleted file mode 100644 index 6e671075..00000000 --- a/content/cover/pkg.go +++ /dev/null @@ -1,17 +0,0 @@ -package size - -func Size(a int) string { - switch { - case a < 0: - return "negative" - case a == 0: - return "zero" - case a < 10: - return "small" - case a < 100: - return "big" - case a < 1000: - return "huge" - } - return "enormous" -} diff --git a/content/cover/pkg_test.go b/content/cover/pkg_test.go deleted file mode 100644 index 8bf4b0cd..00000000 --- a/content/cover/pkg_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package size - -import "testing" - -type Test struct { - in int - out string -} - -var tests = []Test{ - {-1, "negative"}, - {5, "small"}, -} - -func TestSize(t *testing.T) { - for i, test := range tests { - size := Size(test.in) - if size != test.out { - t.Errorf("#%d: Size(%d)=%s; want %s", i, test.in, size, test.out) - } - } -} diff --git a/content/cover/set.png b/content/cover/set.png deleted file mode 100644 index d4cfca2a..00000000 Binary files a/content/cover/set.png and /dev/null differ diff --git a/content/debugging-go-code-status-report.article b/content/debugging-go-code-status-report.article deleted file mode 100644 index 0706d14f..00000000 --- a/content/debugging-go-code-status-report.article +++ /dev/null @@ -1,34 +0,0 @@ -Debugging Go code (a status report) -2 Nov 2010 -Tags: debug, gdb - -Luuk van Dijk - -* Introduction - -When it comes to debugging, nothing beats a few strategic print statements to inspect variables or a well-placed panic to obtain a stack trace. However, sometimes you’re missing either the patience or the source code, and in those cases a good debugger can be invaluable. That's why over the past few releases we have been improving the support in Go’s gc linker (6l, 8l) for GDB, the GNU debugger. - -In the latest release (2010-11-02), the 6l and 8l linkers emit DWARF3 debugging information when writing ELF (Linux, FreeBSD) or Mach-O (Mac OS X) binaries. The DWARF code is rich enough to let you do the following: - -- load a Go program in GDB version 7.x, -- list all Go, C, and assembly source files by line (parts of the Go runtime are written in C and assembly), -- set breakpoints by line and step through the code, -- print stack traces and inspect stack frames, and -- find the addresses and print the contents of most variables. - -There are still some inconveniences: - -- The emitted DWARF code is unreadable by the GDB version 6.x that ships with Mac OS X. We would gladly accept patches to make the DWARF output compatible with the standard OS X GDB, but until that’s fixed you’ll need to download, build, and install GDB 7.x to use it under OS X. The source can be found at [[http://sourceware.org/gdb/download/][http://sourceware.org/gdb/download/]]. Due to the particulars of OS X you’ll need to install the binary on a local file system with `chgrp`procmod` and `chmod`g+s`. -- Names are qualified with a package name and, as GDB doesn't understand Go packages, you must reference each item by its full name. For example, the variable named `v` in package `main` must be referred to as `'main.v'`, in single quotes. A consequence of this is that tab completion of variable and function names does not work. -- Lexical scoping information is somewhat obfuscated. If there are multiple variables of the same name, the nth instance will have a suffix of the form ‘#n’. We plan to fix this, but it will require some changes to the data exchanged between the compiler and linker. -- Slice and string variables are represented as their underlying structure in the runtime library. They will look something like `{data`=`0x2aaaaab3e320,`len`=`1,`cap`=`1}.` For slices, you must dereference the data pointer to inspect the elements. - -Some things don't work yet: - -- Channel, function, interface, and map variables cannot be inspected. -- Only Go variables are annotated with type information; the runtime's C variables are not. -- Windows and ARM binaries do not contain DWARF debugging information and, as such, cannot be inspected with GDB. - -Over the coming months we intend to address these issues, either by changing the compiler and linker or by using the Python extensions to GDB. In the meantime, we hope that Go programmers will benefit from having better access to this well-known debugging tool. - -P.S. The DWARF information can also be read by tools other than GDB. For example, on Linux you can use it with the sysprof system-wide profiler. diff --git a/content/debugging-go-programs-with-gnu-debugger.article b/content/debugging-go-programs-with-gnu-debugger.article deleted file mode 100644 index 1ff9b279..00000000 --- a/content/debugging-go-programs-with-gnu-debugger.article +++ /dev/null @@ -1,12 +0,0 @@ -Debugging Go programs with the GNU Debugger -30 Oct 2011 -Tags: debug, gdb, technical - -Andrew Gerrand - -* Introduction - -Last year we [[http://blog.golang.org/2010/11/debugging-go-code-status-report.html][reported]] that Go's [[http://golang.org/cmd/gc/][gc]]/[[http://golang.org/cmd/6l/][ld]] toolchain produces DWARFv3 debugging information that can be read by the GNU Debugger (GDB). Since then, work has continued steadily on improving support for debugging Go code with GDB. -Among the improvements are the ability to inspect goroutines and to print native Go data types, including structs, slices, strings, maps, interfaces, and channels. - -To learn more about Go and GDB, see the [[http://golang.org/doc/debugging_with_gdb.html][Debugging with GDB]] article. diff --git a/content/defer-panic-and-recover.article b/content/defer-panic-and-recover.article deleted file mode 100644 index f43383a6..00000000 --- a/content/defer-panic-and-recover.article +++ /dev/null @@ -1,197 +0,0 @@ -Defer, Panic, and Recover -4 Aug 2010 -Tags: defer, panic, recover, technical, function - -Andrew Gerrand - -* Introduction - -Go has the usual mechanisms for control flow: if, for, switch, goto. It also has the go statement to run code in a separate goroutine. Here I'd like to discuss some of the less common ones: defer, panic, and recover. - - -A *defer*statement* pushes a function call onto a list. The list of saved calls is executed after the surrounding function returns. Defer is commonly used to simplify functions that perform various clean-up actions. - - -For example, let's look at a function that opens two files and copies the contents of one file to the other: - - - func CopyFile(dstName, srcName string) (written int64, err error) { - src, err := os.Open(srcName) - if err != nil { - return - } - - dst, err := os.Create(dstName) - if err != nil { - return - } - - written, err = io.Copy(dst, src) - dst.Close() - src.Close() - return - } - -This works, but there is a bug. If the call to os.Create fails, the function will return without closing the source file. This can be easily remedied by putting a call to src.Close before the second return statement, but if the function were more complex the problem might not be so easily noticed and resolved. By introducing defer statements we can ensure that the files are always closed: - - - func CopyFile(dstName, srcName string) (written int64, err error) { - src, err := os.Open(srcName) - if err != nil { - return - } - defer src.Close() - - dst, err := os.Create(dstName) - if err != nil { - return - } - defer dst.Close() - - return io.Copy(dst, src) - } - -Defer statements allow us to think about closing each file right after opening it, guaranteeing that, regardless of the number of return statements in the function, the files _will_ be closed. - -The behavior of defer statements is straightforward and predictable. There are three simple rules: - -1. _A_deferred_function's_arguments_are_evaluated_when_the_defer_statement_is_evaluated._ - -In this example, the expression "i" is evaluated when the Println call is deferred. The deferred call will print "0" after the function returns. - - - func a() { - i := 0 - defer fmt.Println(i) - i++ - return - } - -2. _Deferred_function_calls_are_executed_in_Last_In_First_Out_order_after_the_surrounding_function_returns._ - -This function prints "3210": - - func b() { - for i := 0; i < 4; i++ { - defer fmt.Print(i) - } - } - - -3. _Deferred_functions_may_read_and_assign_to_the_returning_function's_named_return_values._ - -In this example, a deferred function increments the return value i _after_ the surrounding function returns. Thus, this function returns 2: - - func c() (i int) { - defer func() { i++ }() - return 1 - } - - -This is convenient for modifying the error return value of a function; we will see an example of this shortly. - - -*Panic* is a built-in function that stops the ordinary flow of control and begins _panicking_. When the function F calls panic, execution of F stops, any deferred functions in F are executed normally, and then F returns to its caller. To the caller, F then behaves like a call to panic. The process continues up the stack until all functions in the current goroutine have returned, at which point the program crashes. Panics can be initiated by invoking panic directly. They can also be caused by runtime errors, such as out-of-bounds array accesses. - - -*Recover* is a built-in function that regains control of a panicking goroutine. Recover is only useful inside deferred functions. During normal execution, a call to recover will return nil and have no other effect. If the current goroutine is panicking, a call to recover will capture the value given to panic and resume normal execution. - - -Here's an example program that demonstrates the mechanics of panic and defer: - - package main - - import "fmt" - - func main() { - f() - fmt.Println("Returned normally from f.") - } - - func f() { - defer func() { - if r := recover(); r != nil { - fmt.Println("Recovered in f", r) - } - }() - fmt.Println("Calling g.") - g(0) - fmt.Println("Returned normally from g.") - } - - func g(i int) { - if i > 3 { - fmt.Println("Panicking!") - panic(fmt.Sprintf("%v", i)) - } - defer fmt.Println("Defer in g", i) - fmt.Println("Printing in g", i) - g(i + 1) - } - - -The function g takes the int i, and panics if i is greater than 3, or else it calls itself with the argument i+1. The function f defers a function that calls recover and prints the recovered value (if it is non-nil). Try to picture what the output of this program might be before reading on. - - -The program will output: - - - Calling g. - Printing in g 0 - Printing in g 1 - Printing in g 2 - Printing in g 3 - Panicking! - Defer in g 3 - Defer in g 2 - Defer in g 1 - Defer in g 0 - Recovered in f 4 - Returned normally from f. - - - -If we remove the deferred function from f the panic is not recovered and reaches the top of the goroutine's call stack, terminating the program. This modified program will output: - - - Calling g. - Printing in g 0 - Printing in g 1 - Printing in g 2 - Printing in g 3 - Panicking! - Defer in g 3 - Defer in g 2 - Defer in g 1 - Defer in g 0 - panic: 4 - - panic PC=0x2a9cd8 - [stack trace omitted] - - - -For a real-world example of *panic* and *recover*, see the [[http://golang.org/pkg/encoding/json/][json package]] from the Go standard library. It decodes JSON-encoded data with a set of recursive functions. When malformed JSON is encountered, the parser calls panic to unwind the stack to the top-level function call, which recovers from the panic and returns an appropriate error value (see the 'error' and 'unmarshal' methods of the decodeState type in [[http://golang.org/src/pkg/encoding/json/decode.go][decode.go]]). - -The convention in the Go libraries is that even when a package uses panic internally, its external API still presents explicit error return values. - - -Other uses of *defer* (beyond the file.Close example given earlier) include releasing a mutex: - - mu.Lock() - defer mu.Unlock() - - - -printing a footer: - - - printHeader() - defer printFooter() - - - -and more. - - -In summary, the defer statement (with or without panic and recover) provides an unusual and powerful mechanism for control flow. It can be used to model a number of features implemented by special-purpose structures in other programming languages. Try it out. diff --git a/content/docker-outyet.png b/content/docker-outyet.png deleted file mode 100644 index c5e57103..00000000 Binary files a/content/docker-outyet.png and /dev/null differ diff --git a/content/docker.article b/content/docker.article deleted file mode 100644 index 942bcf10..00000000 --- a/content/docker.article +++ /dev/null @@ -1,201 +0,0 @@ -Deploying Go servers with Docker -26 Sep 2014 - -Andrew Gerrand - -* Introduction - -This week Docker [[https://blog.docker.com/2014/09/docker-hub-official-repos-announcing-language-stacks/][announced]] -official base images for Go and other major languages, -giving programmers a trusted and easy way to build containers for their Go programs. - -In this article we'll walk through a recipe for creating a Docker container for -a simple Go web application and deploying that container to Google Compute Engine. -If you're not familiar with Docker, you should read -[[http://docs.docker.com/introduction/understanding-docker/][Understanding Docker]] -before reading on. - -* The demo app - -For our demonstration we will use the -[[http://godoc.org/github.com/golang/example/outyet][outyet]] program from the -[[https://github.com/golang/example][Go examples repository]], -a simple web server that reports whether the next version of Go has been released -(designed to power sites like [[http://isgo1point4.outyet.org/][isgo1point4.outyet.org]]). -It has no dependencies outside the standard library and requires no additional -data files at run time; for a web server, it's about as simple as it gets. - -Use "go get" to fetch and install outyet in your -[[http://golang.org/doc/code.html#Workspaces][workspace]]: - - $ go get github.com/golang/example/outyet - -* Write a Dockerfile - -Replace a file named `Dockerfile` in the `outyet` directory with the following contents: - - # Start from a Debian image with the latest version of Go installed - # and a workspace (GOPATH) configured at /go. - FROM golang - - # Copy the local package files to the container's workspace. - ADD . /go/src/github.com/golang/example/outyet - - # Build the outyet command inside the container. - # (You may fetch or manage dependencies here, - # either manually or with a tool like "godep".) - RUN go install github.com/golang/example/outyet - - # Run the outyet command by default when the container starts. - ENTRYPOINT /go/bin/outyet - - # Document that the service listens on port 8080. - EXPOSE 8080 - -This `Dockerfile` specifies how to construct a container that runs `outyet`, -starting with the basic dependencies (a Debian system with Go installed; -the [[https://registry.hub.docker.com/_/golang/][official `golang` docker image]]), -adding the `outyet` package source, building it, and then finally running it. - -The `ADD`, `RUN`, and `ENTRYPOINT` steps are common tasks for any Go project. -To simplify this, there is an -[[https://github.com/docker-library/golang/blob/9ff2ccca569f9525b023080540f1bb55f6b59d7f/1.3.1/onbuild/Dockerfile][`onbuild` variant]] -of the `golang` image that automatically copies the package source, fetches the -application dependencies, builds the program, and configures it to run on -startup. - -With the `onbuild` variant, the `Dockerfile` is much simpler: - - FROM golang:onbuild - EXPOSE 8080 - -* Build and run the image - -Invoke Docker from the `outyet` package directory to build an image using the `Dockerfile`: - - $ docker build -t outyet . - -This will fetch the `golang` base image from Docker Hub, copy the package source -to it, build the package inside it, and tag the resulting image as `outyet`. - -To run a container from the resulting image: - - $ docker run --publish 6060:8080 --name test --rm outyet - -The `--publish` flag tells docker to publish the container's port `8080` on the -external port `6060`. - -The `--name` flag gives our container a predictable name to make it easier to work with. - -The `--rm` flag tells docker to remove the container image when the outyet server exits. - -With the container running, open `http://localhost:6060/` in a web browser and -you should see something like this: - -.image docker-outyet.png - -(If your docker daemon is running on another machine (or in a virtual machine), -you should replace `localhost` with the address of that machine. If you're -using [[http://boot2docker.io/][boot2docker]] on OS X or Windows you can find -that address with `boot2docker`ip`.) - -Now that we've verified that the image works, shut down the running container -from another terminal window: - - $ docker stop test - -* Create a repository on Docker Hub - -[[https://hub.docker.com/][Docker Hub]], the container registry from which we -pulled the `golang` image earlier, offers a feature called -[[http://docs.docker.com/docker-hub/builds/][Automated Builds]] that builds -images from a GitHub or BitBucket repository. - -By committing [[https://github.com/golang/example/blob/master/outyet/Dockerfile][the Dockerfile]] -to the repository and creating an -[[https://registry.hub.docker.com/u/adg1/outyet/][automated build]] -for it, anyone with Docker installed can download and run our image with a -single command. (We will see the utility of this in the next section.) - -To set up an Automated Build, commit the Dockerfile to your repo on -[[https://github.com/][GitHub]] or [[https://bitbucket.org/][BitBucket]], -create an account on Docker Hub, and follow the instructions for -[[http://docs.docker.com/docker-hub/builds/][creating an Automated Build]]. - -When you're done, you can run your container using the name of the automated build: - - $ docker run goexample/outyet - -(Replace `goexample/outyet` with the name of the automated build you created.) - -* Deploy the container to Google Compute Engine - -Google provides -[[https://developers.google.com/compute/docs/containers/container_vms][container-optimized Google Compute Engine images]] -that make it easy to spin up a virtual machine running an arbitrary Docker container. -On startup, a program running on the instance reads a configuration file that -specifies which container to run, fetches the container image, and runs it. - -Create a [[https://cloud.google.com/compute/docs/containers/container_vms#container_manifest][containers.yaml]] -file that specifies the docker image to run and the ports to expose: - - version: v1beta2 - containers: - - name: outyet - image: goexample/outyet - ports: - - name: http - hostPort: 80 - containerPort: 8080 - -(Note that we're publishing the container's port `8080` as external port `80`, -the default port for serving HTTP traffic. And, again, you should replace -`goexample/outyet` with the name of your Automated Build.) - -Use the [[https://cloud.google.com/sdk/#Quick_Start][gcloud tool]] -to create a VM instance running the container: - - $ gcloud compute instances create outyet \ - --image container-vm-v20140925 \ - --image-project google-containers \ - --metadata-from-file google-container-manifest=containers.yaml \ - --tags http-server \ - --zone us-central1-a \ - --machine-type f1-micro - -The first argument (`outyet`) specifies the instance name, a convenient label -for administrative purposes. - -The `--image` and `--image-project` flags specify the special -container-optimized system image to use (copy these flags verbatim). - -The `--metadata-from-file` flag supplies your `containers.yaml` file to the VM. - -The `--tags` flag tags your VM instance as an HTTP server, adjusting the -firewall to expose port 80 on the public network interface. - -The `--zone` and `--machine-type` flags specify the zone in which to run the VM -and the type of machine to run. (To see a list of machine types and the zones, -run `gcloud`compute`machine-types`list`.) - -Once this has completed, the gcloud command should print some information about -the instance. In the output, locate the `networkInterfaces` section to find the -instance's external IP address. Within a couple of minutes you should be able -to access that IP with your web browser and see the "Has Go 1.4 been released -yet?" page. - -(To see what's happening on the new VM instance you can ssh into it with -`gcloud`compute`ssh`outyet`. From there, try `sudo`docker`ps` to see which -Docker containers are running.) - -* Learn more - -This is just the tip of the iceberg—there's a lot more you can do with Go, Docker, and Google Compute Engine. - -To learn more about Docker, see their [[https://docs.docker.com/][extensive documentation]]. - -To learn more about Docker and Go, see the [[https://registry.hub.docker.com/_/golang/][official `golang` Docker Hub repository]] and Kelsey Hightower's [[https://medium.com/@kelseyhightower/optimizing-docker-images-for-static-binaries-b5696e26eb07][Optimizing Docker Images for Static Go Binaries]]. - -To learn more about Docker and [[http://cloud.google.com/compute][Google Compute Engine]], -see the [[https://cloud.google.com/compute/docs/containers/container_vms][Container-optimized VMs page]] -and the [[https://registry.hub.docker.com/u/google/docker-registry/][google/docker-registry Docker Hub repository]]. diff --git a/content/error-handling-and-go.article b/content/error-handling-and-go.article deleted file mode 100644 index 266458f7..00000000 --- a/content/error-handling-and-go.article +++ /dev/null @@ -1,242 +0,0 @@ -Error handling and Go -12 Jul 2011 -Tags: error, interface, type, technical - -Andrew Gerrand - -* Introduction - -If you have written any Go code you have probably encountered the built-in `error` type. Go code uses `error` values to indicate an abnormal state. For example, the `os.Open` function returns a non-nil `error` value when it fails to open a file. - - func Open(name string) (file *File, err error) - -The following code uses `os.Open` to open a file. If an error occurs it calls `log.Fatal` to print the error message and stop. - - f, err := os.Open("filename.ext") - if err != nil { - log.Fatal(err) - } - // do something with the open *File f - -You can get a lot done in Go knowing just this about the `error` type, but in this article we'll take a closer look at `error` and discuss some good practices for error handling in Go. - -* The error type - -The `error` type is an interface type. An `error` variable represents any value that can describe itself as a string. Here is the interface's declaration: - - type error interface { - Error() string - } - -The `error` type, as with all built in types, is [[http://golang.org/doc/go_spec.html#Predeclared_identifiers][predeclared]] in the [[http://golang.org/doc/go_spec.html#Blocks][universe block]]. - -The most commonly-used `error` implementation is the [[http://golang.org/pkg/errors/][errors]] package's unexported `errorString` type. - - // errorString is a trivial implementation of error. - type errorString struct { - s string - } - - func (e *errorString) Error() string { - return e.s - } - -You can construct one of these values with the `errors.New` function. It takes a string that it converts to an `errors.errorString` and returns as an `error` value. - - // New returns an error that formats as the given text. - func New(text string) error { - return &errorString{text} - } - -Here's how you might use `errors.New`: - - func Sqrt(f float64) (float64, error) { - if f < 0 { - return 0, errors.New("math: square root of negative number") - } - // implementation - } - -A caller passing a negative argument to `Sqrt` receives a non-nil `error` value (whose concrete representation is an `errors.errorString` value). The caller can access the error string ("math: square root of...") by calling the `error`'s `Error` method, or by just printing it: - - f, err := Sqrt(-1) - if err != nil { - fmt.Println(err) - } - -The [[http://golang.org/pkg/fmt/][fmt]] package formats an `error` value by calling its `Error()`string` method. - -It is the error implementation's responsibility to summarize the context. The error returned by `os.Open` formats as "open /etc/passwd: permission denied," not just "permission denied." The error returned by our `Sqrt` is missing information about the invalid argument. - -To add that information, a useful function is the `fmt` package's `Errorf`. It formats a string according to `Printf`'s rules and returns it as an `error` created by `errors.New`. - - if f < 0 { - return 0, fmt.Errorf("math: square root of negative number %g", f) - } - -In many cases `fmt.Errorf` is good enough, but since `error` is an interface, you can use arbitrary data structures as error values, to allow callers to inspect the details of the error. - -For instance, our hypothetical callers might want to recover the invalid argument passed to `Sqrt`. We can enable that by defining a new error implementation instead of using `errors.errorString`: - - type NegativeSqrtError float64 - - func (f NegativeSqrtError) Error() string { - return fmt.Sprintf("math: square root of negative number %g", float64(f)) - } - -A sophisticated caller can then use a [[http://golang.org/doc/go_spec.html#Type_assertions][type assertion]] to check for a `NegativeSqrtError` and handle it specially, while callers that just pass the error to `fmt.Println` or `log.Fatal` will see no change in behavior. - -As another example, the [[http://golang.org/pkg/encoding/json/][json]] package specifies a `SyntaxError` type that the `json.Decode` function returns when it encounters a syntax error parsing a JSON blob. - - type SyntaxError struct { - msg string // description of error - Offset int64 // error occurred after reading Offset bytes - } - - func (e *SyntaxError) Error() string { return e.msg } - -The `Offset` field isn't even shown in the default formatting of the error, but callers can use it to add file and line information to their error messages: - - if err := dec.Decode(&val); err != nil { - if serr, ok := err.(*json.SyntaxError); ok { - line, col := findLine(f, serr.Offset) - return fmt.Errorf("%s:%d:%d: %v", f.Name(), line, col, err) - } - return err - } - -(This is a slightly simplified version of some [[http://camlistore.org/code/?p=camlistore.git;a=blob;f=lib/go/camli/jsonconfig/eval.go#l68][actual code]] from the [[http://camlistore.org][Camlistore]] project.) - -The `error` interface requires only a `Error` method; specific error implementations might have additional methods. For instance, the [[http://golang.org/pkg/net/][net]] package returns errors of type `error`, following the usual convention, but some of the error implementations have additional methods defined by the `net.Error` interface: - - package net - - type Error interface { - error - Timeout() bool // Is the error a timeout? - Temporary() bool // Is the error temporary? - } - -Client code can test for a `net.Error` with a type assertion and then distinguish transient network errors from permanent ones. For instance, a web crawler might sleep and retry when it encounters a temporary error and give up otherwise. - - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - time.Sleep(1e9) - continue - } - if err != nil { - log.Fatal(err) - } - -* Simplifying repetitive error handling - -In Go, error handling is important. The language's design and conventions encourage you to explicitly check for errors where they occur (as distinct from the convention in other languages of throwing exceptions and sometimes catching them). In some cases this makes Go code verbose, but fortunately there are some techniques you can use to minimize repetitive error handling. - -Consider an [[http://code.google.com/appengine/docs/go/][App Engine]] application with an HTTP handler that retrieves a record from the datastore and formats it with a template. - - func init() { - http.HandleFunc("/view", viewRecord) - } - - func viewRecord(w http.ResponseWriter, r *http.Request) { - c := appengine.NewContext(r) - key := datastore.NewKey(c, "Record", r.FormValue("id"), 0, nil) - record := new(Record) - if err := datastore.Get(c, key, record); err != nil { - http.Error(w, err.Error(), 500) - return - } - if err := viewTemplate.Execute(w, record); err != nil { - http.Error(w, err.Error(), 500) - } - } - -This function handles errors returned by the `datastore.Get` function and `viewTemplate`'s `Execute` method. In both cases, it presents a simple error message to the user with the HTTP status code 500 ("Internal Server Error"). This looks like a manageable amount of code, but add some more HTTP handlers and you quickly end up with many copies of identical error handling code. - -To reduce the repetition we can define our own HTTP `appHandler` type that includes an `error` return value: - - type appHandler func(http.ResponseWriter, *http.Request) error - -Then we can change our `viewRecord` function to return errors: - - func viewRecord(w http.ResponseWriter, r *http.Request) error { - c := appengine.NewContext(r) - key := datastore.NewKey(c, "Record", r.FormValue("id"), 0, nil) - record := new(Record) - if err := datastore.Get(c, key, record); err != nil { - return err - } - return viewTemplate.Execute(w, record) - } - -This is simpler than the original version, but the [[http://golang.org/pkg/net/http/][http]] package doesn't understand functions that return `error`. To fix this we can implement the `http.Handler` interface's `ServeHTTP` method on `appHandler`: - - func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if err := fn(w, r); err != nil { - http.Error(w, err.Error(), 500) - } - } - -The `ServeHTTP` method calls the `appHandler` function and displays the returned error (if any) to the user. Notice that the method's receiver, `fn`, is a function. (Go can do that!) The method invokes the function by calling the receiver in the expression `fn(w,`r)`. - -Now when registering `viewRecord` with the http package we use the `Handle` function (instead of `HandleFunc`) as `appHandler` is an `http.Handler` (not an `http.HandlerFunc`). - - func init() { - http.Handle("/view", appHandler(viewRecord)) - } - -With this basic error handling infrastructure in place, we can make it more user friendly. Rather than just displaying the error string, it would be better to give the user a simple error message with an appropriate HTTP status code, while logging the full error to the App Engine developer console for debugging purposes. - -To do this we create an `appError` struct containing an `error` and some other fields: - - type appError struct { - Error error - Message string - Code int - } - -Next we modify the appHandler type to return `*appError` values: - - type appHandler func(http.ResponseWriter, *http.Request) *appError - -(It's usually a mistake to pass back the concrete type of an error rather than `error`, for reasons discussed in [[http://golang.org/doc/go_faq.html#nil_error][the Go FAQ]], but it's the right thing to do here because `ServeHTTP` is the only place that sees the value and uses its contents.) - -And make `appHandler`'s `ServeHTTP` method display the `appError`'s `Message` to the user with the correct HTTP status `Code` and log the full `Error` to the developer console: - - func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if e := fn(w, r); e != nil { // e is *appError, not os.Error. - c := appengine.NewContext(r) - c.Errorf("%v", e.Error) - http.Error(w, e.Message, e.Code) - } - } - -Finally, we update `viewRecord` to the new function signature and have it return more context when it encounters an error: - - func viewRecord(w http.ResponseWriter, r *http.Request) *appError { - c := appengine.NewContext(r) - key := datastore.NewKey(c, "Record", r.FormValue("id"), 0, nil) - record := new(Record) - if err := datastore.Get(c, key, record); err != nil { - return &appError{err, "Record not found", 404} - } - if err := viewTemplate.Execute(w, record); err != nil { - return &appError{err, "Can't display record", 500} - } - return nil - } - -This version of `viewRecord` is the same length as the original, but now each of those lines has specific meaning and we are providing a friendlier user experience. - -It doesn't end there; we can further improve the error handling in our application. Some ideas: - -- give the error handler a pretty HTML template, - -- make debugging easier by writing the stack trace to the HTTP response when the user is an administrator, - -- write a constructor function for `appError` that stores the stack trace for easier debugging, - -- recover from panics inside the `appHandler`, logging the error to the console as "Critical," while telling the user "a serious error has occurred." This is a nice touch to avoid exposing the user to inscrutable error messages caused by programming errors. See the [[http://golang.org/doc/articles/defer_panic_recover.html][Defer, Panic, and Recover]] article for more details. - -* Conclusion - -Proper error handling is an essential requirement of good software. By employing the techniques described in this post you should be able to write more reliable and succinct Go code. diff --git a/content/errors-are-values.article b/content/errors-are-values.article deleted file mode 100644 index 95a7d3f0..00000000 --- a/content/errors-are-values.article +++ /dev/null @@ -1,235 +0,0 @@ -Errors are values -12 Jan 2015 - -Rob Pike - -* Introduction - -A common point of discussion among Go programmers, -especially those new to the language, is how to handle errors. -The conversation often turns into a lament at the number of times the sequence - - if err != nil { - return err - } - -shows up. -We recently scanned all the open source projects we could find and -discovered that this snippet occurs only once per page or two, -less often than some would have you believe. -Still, if the perception persists that one must type - - if err != nil - -all the time, something must be wrong, and the obvious target is Go itself. - -This is unfortunate, misleading, and easily corrected. -Perhaps what is happening is that programmers new to Go ask, -"How does one handle errors?", learn this pattern, and stop there. -In other languages, one might use a try-catch block or other such mechanism to handle errors. -Therefore, the programmer thinks, when I would have used a try-catch -in my old language, I will just type `if` `err` `!=` `nil` in Go. -Over time the Go code collects many such snippets, and the result feels clumsy. - -Regardless of whether this explanation fits, -it is clear that these Go programmers miss a fundamental point about errors: -_Errors_are_values._ - -Values can be programmed, and since errors are values, errors can be programmed. - -Of course a common statement involving an error value is to test whether it is nil, -but there are countless other things one can do with an error value, -and application of some of those other things can make your program better, -eliminating much of the boilerplate that arises if every error is checked with a rote if statement. - -Here's a simple example from the `bufio` package's -[[http://golang.org/pkg/bufio/#Scanner][`Scanner`]] type. -Its [[http://golang.org/pkg/bufio/#Scanner.Scan][`Scan`]] method performs the underlying I/O, -which can of course lead to an error. -Yet the `Scan` method does not expose an error at all. -Instead, it returns a boolean, and a separate method, to be run at the end of the scan, -reports whether an error occurred. -Client code looks like this: - - scanner := bufio.NewScanner(input) - for scanner.Scan() { - token := scanner.Text() - // process token - } - if err := scanner.Err(); err != nil { - // process the error - } - -Sure, there is a nil check for an error, but it appears and executes only once. -The `Scan` method could instead have been defined as - - func (s *Scanner) Scan() (token []byte, error) - -and then the example user code might be (depending on how the token is retrieved), - - scanner := bufio.NewScanner(input) - for { - token, err := scanner.Scan() - if err != nil { - return err // or maybe break - } - // process token - } - -This isn't very different, but there is one important distinction. -In this code, the client must check for an error on every iteration, -but in the real `Scanner` API, the error handling is abstracted away from the key API element, -which is iterating over tokens. -With the real API, the client's code therefore feels more natural: -loop until done, then worry about errors. -Error handling does not obscure the flow of control. - -Under the covers what's happening, of course, -is that as soon as `Scan` encounters an I/O error, it records it and returns `false`. -A separate method, [[http://golang.org/pkg/bufio/#Scanner.Err][`Err`]], -reports the error value when the client asks. -Trivial though this is, it's not the same as putting - - if err != nil - -everywhere or asking the client to check for an error after every token. -It's programming with error values. -Simple programming, yes, but programming nonetheless. - -It's worth stressing that whatever the design, -it's critical that the program check the errors however they are exposed. -The discussion here is not about how to avoid checking errors, -it's about using the language to handle errors with grace. - -The topic of repetitive error-checking code arose when I attended the autumn 2014 GoCon in Tokyo. -An enthusiastic gopher, who goes by [[https://twitter.com/jxck_][`@jxck_`]] on Twitter, -echoed the familiar lament about error checking. -He had some code that looked schematically like this: - - _, err = fd.Write(p0[a:b]) - if err != nil { - return err - } - _, err = fd.Write(p1[c:d]) - if err != nil { - return err - } - _, err = fd.Write(p2[e:f]) - if err != nil { - return err - } - // and so on - -It is very repetitive. -In the real code, which was longer, -there is more going on so it's not easy to just refactor this using a helper function, -but in this idealized form, a function literal closing over the error variable would help: - - var err error - write := func(buf []byte) { - if err != nil { - return - } - _, err = w.Write(buf) - } - write(p0[a:b]) - write(p1[c:d]) - write(p2[e:f]) - // and so on - if err != nil { - return err - } - -This pattern works well, but requires a closure in each function doing the writes; -a separate helper function is clumsier to use because the `err` variable -needs to be maintained across calls (try it). - -We can make this cleaner, more general, and reusable by borrowing the idea from the -`Scan` method above. -I mentioned this technique in our discussion but `@jxck_` didn't see how to apply it. -After a long exchange, hampered somewhat by a language barrier, -I asked if I could just borrow his laptop and show him by typing some code. - -I defined an object called an `errWriter`, something like this: - - type errWriter struct { - w io.Writer - err error - } - -and gave it one method, `write.` -It doesn't need to have the standard `Write` signature, -and it's lower-cased in part to highlight the distinction. -The `write` method calls the `Write` method of the underlying `Writer` -and records the first error for future reference: - - func (ew *errWriter) write(buf []byte) { - if ew.err != nil { - return - } - _, ew.err = ew.w.Write(buf) - } - -As soon as an error occurs, the `write` method becomes a no-op but the error value is saved. - -Given the `errWriter` type and its `write` method, the code above can be refactored: - - ew := &errWriter{w: fd} - ew.write(p0[a:b]) - ew.write(p1[c:d]) - ew.write(p2[e:f]) - // and so on - if ew.err != nil { - return ew.err - } - -This is cleaner, even compared to the use of a closure, -and also makes the actual sequence of writes being done easier to see on the page. -There is no clutter any more. -Programming with error values (and interfaces) has made the code nicer. - -It's likely that some other piece of code in the same package can build on this idea, -or even use `errWriter` directly. - -Also, once `errWriter` exists, there's more it could do to help, -especially in less artificial examples. -It could accumulate the byte count. -It could coalesce writes into a single buffer that can then be transmitted atomically. -And much more. - -In fact, this pattern appears often in the standard library. -The [[http://golang.org/pkg/archive/zip/][`archive/zip`]] and -[[http://golang.org/pkg/net/http/][`net/http`]] packages use it. -More salient to this discussion, the [[http://golang.org/pkg/bufio/][`bufio` package's `Writer`]] -is actually an implementation of the `errWriter` idea. -Although `bufio.Writer.Write` returns an error, -that is mostly about honoring the [[http://golang.org/pkg/io/#Writer][`io.Writer`]] interface. -The `Write` method of `bufio.Writer` behaves just like our `errWriter.write` -method above, with `Flush` reporting the error, so our example could be written like this: - - b := bufio.NewWriter(fd) - b.Write(p0[a:b]) - b.Write(p1[c:d]) - b.Write(p2[e:f]) - // and so on - if b.Flush() != nil { - return b.Flush() - } - -There is one significant drawback to this approach, at least for some applications: -there is no way to know how much of the processing completed before the error occurred. -If that information is important, a more fine-grained approach is necessary. -Often, though, an all-or-nothing check at the end is sufficient. - -We've looked at just one technique for avoiding repetitive error handling code. -Keep in mind that the use of `errWriter` or `bufio.Writer` isn't the only way to simplify error handling, -and this approach is not suitable for all situations. -The key lesson, however, is that errors are values and the full power of -the Go programming language is available for processing them. - -Use the language to simplify your error handling. - -But remember: Whatever you do, always check your errors! - -Finally, for the full story of my interaction with @jxck_, including a little video he recorded, -visit [[http://jxck.hatenablog.com/entry/golang-error-handling-lesson-by-rob-pike][his blog]]. diff --git a/content/examples.article b/content/examples.article deleted file mode 100644 index aa0dd286..00000000 --- a/content/examples.article +++ /dev/null @@ -1,201 +0,0 @@ -Testable Examples in Go -7 May 2015 -Tags: godoc, testing - -Andrew Gerrand - -* Introduction - -Godoc [[http://golang.org/pkg/testing/#hdr-Examples][examples]] are snippets of -Go code that are displayed as package documentation and that are verified by -running them as tests. -They can also be run by a user visiting the godoc web page for the package -and clicking the associated "Run" button. - -Having executable documentation for a package guarantees that the information -will not go out of date as the API changes. - -The standard library includes many such examples -(see the [[http://golang.org/pkg/strings/#Contains][`strings` package]], -for instance). - -This article explains how to write your own example functions. - -* Examples are tests - -Examples are compiled (and optionally executed) as part of a package's test -suite. - -As with typical tests, examples are functions that reside in a package's -`_test.go` files. -Unlike normal test functions, though, example functions take no arguments -and begin with the word `Example` instead of `Test`. - -The [[https://godoc.org/github.com/golang/example/stringutil/][`stringutil` package]] -is part of the [[https://github.com/golang/example][Go example repository]]. -Here's an example that demonstrates its `Reverse` function: - - package stringutil_test - - import ( - "fmt" - - "github.com/golang/example/stringutil" - ) - - func ExampleReverse() { - fmt.Println(stringutil.Reverse("hello")) - // Output: olleh - } - -This code might live in `example_test.go` in the `stringutil` directory. - -Godoc will present this example alongside the `Reverse` function's documentation: - -.image examples/reverse.png - -Running the package's test suite, we can see the example function is executed -with no further arrangement from us: - - $ go test -v - === RUN TestReverse - --- PASS: TestReverse (0.00s) - === RUN: ExampleReverse - --- PASS: ExampleReverse (0.00s) - PASS - ok github.com/golang/example/stringutil 0.009s - -* Output comments - -What does it mean that the `ExampleReverse` function "passes"? - -As it executes the example, -the testing framework captures data written to standard output -and then compares the output against the example's "Output:" comment. -The test passes if the test's output matches its output comment. - -To see a failing example we can change the output comment text to something -obviously incorrect - - func ExampleReverse() { - fmt.Println(stringutil.Reverse("hello")) - // Output: golly - } - -and run the tests again: - - $ go test - --- FAIL: ExampleReverse (0.00s) - got: - olleh - want: - golly - FAIL - -If we remove the output comment entirely - - func ExampleReverse() { - fmt.Println(stringutil.Reverse("hello")) - } - -then the example function is compiled but not executed: - - $ go test -v - === RUN TestReverse - --- PASS: TestReverse (0.00s) - PASS - ok github.com/golang/example/stringutil 0.009s - -Examples without output comments are useful for demonstrating code that cannot -run as unit tests, such as that which accesses the network, -while guaranteeing the example at least compiles. - - -* Example function names - -Godoc uses a naming convention to associate an example function with a -package-level identifier. - - func ExampleFoo() // documents the Foo function or type - func ExampleBar_Qux() // documents the Qux method of type Bar - func Example() // documents the package as a whole - -Following this convention, godoc displays the `ExampleReverse` example -alongside the documentation for the `Reverse` function. - -Multiple examples can be provided for a given identifier by using a suffix -beginning with an underscore followed by a lowercase letter. -Each of these examples documents the `Reverse` function: - - func ExampleReverse() - func ExampleReverse_second() - func ExampleReverse_third() - - -* Larger examples - -Sometimes we need more than just a function to write a good example. - -For instance, to demonstrate the [[https://golang.org/pkg/sort/][`sort` package]] -we should show an implementation of `sort.Interface`. -Since methods cannot be declared inside a function body, the example must -include some context in addition to the example function. - -To achieve this we can use a "whole file example." -A whole file example is a file that ends in `_test.go` and contains exactly one -example function, no test or benchmark functions, and at least one other -package-level declaration. -When displaying such examples godoc will show the entire file. - -Here is a whole file example from the `sort` package: - - package sort_test - - import ( - "fmt" - "sort" - ) - - type Person struct { - Name string - Age int - } - - func (p Person) String() string { - return fmt.Sprintf("%s: %d", p.Name, p.Age) - } - - // ByAge implements sort.Interface for []Person based on - // the Age field. - type ByAge []Person - - func (a ByAge) Len() int { return len(a) } - func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age } - - func Example() { - people := []Person{ - {"Bob", 31}, - {"John", 42}, - {"Michael", 17}, - {"Jenny", 26}, - } - - fmt.Println(people) - sort.Sort(ByAge(people)) - fmt.Println(people) - - // Output: - // [Bob: 31 John: 42 Michael: 17 Jenny: 26] - // [Michael: 17 Jenny: 26 Bob: 31 John: 42] - } - -A package can contain multiple whole file examples; one example per file. -Take a look at the [[https://golang.org/src/sort/][`sort` package's source code]] -to see this in practice. - -* Conclusion - -Godoc examples are a great way to write and maintain code as documentation. -They also present editable, working, runnable examples your users can build on. -Use them! diff --git a/content/examples/reverse.png b/content/examples/reverse.png deleted file mode 100644 index c9b5ad20..00000000 Binary files a/content/examples/reverse.png and /dev/null differ diff --git a/content/first-class-functions-in-go-and-new-go.article b/content/first-class-functions-in-go-and-new-go.article deleted file mode 100644 index 998ccb3d..00000000 --- a/content/first-class-functions-in-go-and-new-go.article +++ /dev/null @@ -1,11 +0,0 @@ -"First Class Functions in Go" -30 Jun 2011 -Tags: codewalk, function, technical - -Andrew Gerrand - -* Introduction - -Programmers new to Go are often surprised by its support for function types, functions as values, and closures. The [[http://golang.org/doc/codewalk/functions/][First Class Functions in Go]] code walk demonstrates these features with a simulation of the dice game [[http://en.wikipedia.org/wiki/Pig_(dice)][Pig]]. It is a pretty program that uses the language to great effect, and a fun read for Go beginners and veterans alike. - -More resources are available at [[http://golang.org/doc/docs.html][golang.org]]. diff --git a/content/first-go-program.article b/content/first-go-program.article deleted file mode 100644 index 3e31585b..00000000 --- a/content/first-go-program.article +++ /dev/null @@ -1,123 +0,0 @@ -The first Go program -18 Jul 2013 -Tags: history - -Andrew Gerrand - -* Introduction - -Brad Fitzpatrick and I (Andrew Gerrand) recently started restructuring -[[http://golang.org/cmd/godoc/][godoc]], and it occurred to me that it is one -of the oldest Go programs. -Robert Griesemer started writing it back in early 2009, -and we're still using it today. - -When I [[https://twitter.com/enneff/status/357403054632484865][tweeted]] about -this, Dave Cheney replied with an [[https://twitter.com/davecheney/status/357406479415914497][interesting question]]: -what is the oldest Go program? Rob Pike dug into his mail and found it -in an old message to Robert and Ken Thompson. - -What follows is the first Go program. It was written by Rob in February 2008, -when the team was just Rob, Robert, and Ken. They had a solid feature list -(mentioned in [[http://commandcenter.blogspot.com.au/2012/06/less-is-exponentially-more.html][this blog post]]) -and a rough language specfication. Ken had just finished the first working version of -a Go compiler (it didn't produce native code, but rather transliterated Go code -to C for fast prototyping) and it was time to try writing a program with it. - -Rob sent mail to the "Go team": - - From: Rob 'Commander' Pike - Date: Wed, Feb 6, 2008 at 3:42 PM - To: Ken Thompson, Robert Griesemer - Subject: slist - - it works now. - - roro=% a.out - (defn foo (add 12 34)) - return: icounter = 4440 - roro=% - - here's the code. - some ugly hackery to get around the lack of strings. - - -(The `icounter` line in the program output is the number of executed -statements, printed for debugging.) - -.code first-go-program/slist.go - -The program parses and prints an -[[https://en.wikipedia.org/wiki/S-expression][S-expression]]. -It takes no user input and has no imports, relying only on the built-in -`print` facility for output. -It was written literally the first day there was a -[[http://golang.org/change/8b8615138da3][working but rudimentary compiler]]. -Much of the language wasn't implemented and some of it wasn't even specified. - -Still, the basic flavor of the language today is recognizable in this program. -Type and variable declarations, control flow, and package statements haven't -changed much. - -But there are many differences and absences. -Most significant are the lack of concurrency and interfaces—both -considered essential since day 1 but not yet designed. - -A `func` was a `function`, and its signature specified return values -_before_ arguments, separating them with `<-`, which we now use as the channel -send/receive operator. For example, the `WhiteSpace` function takes the integer -`c` and returns a boolean. - - function WhiteSpace(bool <- c int) - -This arrow was a stop-gap measure until a better syntax arose for declaring -multiple return values. - -Methods were distinct from functions and had their own keyword. - - method (this *Slist) Car(*Slist <-) { - return this.list.car; - } - -And methods were pre-declared in the struct definition, although that changed soon. - - type Slist struct { - ... - Car method(*Slist <-); - } - -There were no strings, although they were in the spec. -To work around this, Rob had to build the input string as an `uint8` array with -a clumsy construction. (Arrays were rudimentary and slices hadn't been designed -yet, let alone implemented, although there was the unimplemented concept of an -"open array".) - - input[i] = '('; i = i + 1; - input[i] = 'd'; i = i + 1; - input[i] = 'e'; i = i + 1; - input[i] = 'f'; i = i + 1; - input[i] = 'n'; i = i + 1; - input[i] = ' '; i = i + 1; - ... - -Both `panic` and `print` were built-in keywords, not pre-declared functions. - - print "parse error: expected ", c, "\n"; - panic "parse"; - -And there are many other little differences; see if you can identify some others. - -Less than two years after this program was written, Go was released as an -open source project. Looking back, it is striking how much the language has -grown and matured. (The last thing to change between this proto-Go and the Go -we know today was the elimination of semicolons.) - -But even more striking is how much we have learned about _writing_ Go code. -For instance, Rob called his method receivers `this`, but now we use shorter -context-specific names. There are hundreds of more significant examples -and to this day we're still discovering better ways to write Go code. -(Check out the [[https://github.com/golang/glog][glog package]]'s clever trick for -[[https://github.com/golang/glog/blob/c6f9652c7179652e2fd8ed7002330db089f4c9db/glog.go#L893][handling verbosity levels]].) - -I wonder what we'll learn tomorrow. - diff --git a/content/first-go-program/slist.go b/content/first-go-program/slist.go deleted file mode 100644 index bde7e2b7..00000000 --- a/content/first-go-program/slist.go +++ /dev/null @@ -1,322 +0,0 @@ -// +build OMIT - -package main - -// fake stuff -type char uint8; - -// const char TESTSTRING[] = "(defn foo (add 'a 'b))\n"; - -type Atom struct { - string *[100]char; - integer int; - next *Slist; /* in hash bucket */ -} - -type List struct { - car *Slist; - cdr *Slist; -} - -type Slist struct { - isatom bool; - isstring bool; - //union { - atom Atom; - list List; - //} u; - - Free method(); - Print method(); - PrintOne method(doparen bool); - String method(*char <-); - Integer method(int <-); - Car method(*Slist <-); - Cdr method(*Slist <-); -} - -method (this *Slist) Car(*Slist <-) { - return this.list.car; -} - -method (this *Slist) Cdr(*Slist <-) { - return this.list.cdr; -} - -method (this *Slist) String(*[100]char <-) { - return this.atom.string; -} - -method (this *Slist) Integer(int <-) { - return this.atom.integer; -} - -function OpenFile(); -function Parse(*Slist <-); - -//Slist* atom(char *s, int i); - -var token int; -var peekc int = -1; -var lineno int32 = 1; - -var input [100*1000]char; -var inputindex int = 0; -var tokenbuf [100]char; - -var EOF int = -1; // BUG should be const - -function main(int32 <-) { - var list *Slist; - - OpenFile(); - for ;; { - list = Parse(); - if list == nil { - break; - } - list.Print(); - list.Free(); - break; - } - - return 0; -} - -method (slist *Slist) Free(<-) { - if slist == nil { - return; - } - if slist.isatom { -// free(slist.String()); - } else { - slist.Car().Free(); - slist.Cdr().Free(); - } -// free(slist); -} - -method (slist *Slist) PrintOne(<- doparen bool) { - if slist == nil { - return; - } - if slist.isatom { - if slist.isstring { - print(slist.String()); - } else { - print(slist.Integer()); - } - } else { - if doparen { - print("("); - } - slist.Car().PrintOne(true); - if slist.Cdr() != nil { - print(" "); - slist.Cdr().PrintOne(false); - } - if doparen { - print(")"); - } - } -} - -method (slist *Slist) Print() { - slist.PrintOne(true); - print "\n"; -} - -function Get(int <-) { - var c int; - - if peekc >= 0 { - c = peekc; - peekc = -1; - } else { - c = convert(int, input[inputindex]); - inputindex = inputindex + 1; // BUG should be incr one expr - if c == '\n' { - lineno = lineno + 1; - } - if c == '\0' { - inputindex = inputindex - 1; - c = EOF; - } - } - return c; -} - -function WhiteSpace(bool <- c int) { - return c == ' ' || c == '\t' || c == '\r' || c == '\n'; -} - -function NextToken() { - var i, c int; - var backslash bool; - - tokenbuf[0] = '\0'; // clear previous token - c = Get(); - while WhiteSpace(c) { - c = Get(); - } - switch c { - case EOF: - token = EOF; - case '(': - case ')': - token = c; - break; - case: - for i = 0; i < 100 - 1; { // sizeof tokenbuf - 1 - tokenbuf[i] = convert(char, c); - i = i + 1; - c = Get(); - if c == EOF { - break; - } - if WhiteSpace(c) || c == ')' { - peekc = c; - break; - } - } - if i >= 100 - 1 { // sizeof tokenbuf - 1 - panic "atom too long\n"; - } - tokenbuf[i] = '\0'; - if '0' <= tokenbuf[0] && tokenbuf[0] <= '9' { - token = '0'; - } else { - token = 'A'; - } - } -} - -function Expect(<- c int) { - if token != c { - print "parse error: expected ", c, "\n"; - panic "parse"; - } - NextToken(); -} - -// Parse a non-parenthesized list up to a closing paren or EOF -function ParseList(*Slist <-) { - var slist, retval *Slist; - - slist = new(Slist); - slist.list.car = nil; - slist.list.cdr = nil; - slist.isatom = false; - slist.isstring = false; - - retval = slist; - for ;; { - slist.list.car = Parse(); - if token == ')' { // empty cdr - break; - } - if token == EOF { // empty cdr BUG SHOULD USE || - break; - } - slist.list.cdr = new(Slist); - slist = slist.list.cdr; - } - return retval; -} - -function atom(*Slist <- i int) { // BUG: uses tokenbuf; should take argument - var h, length int; - var slist, tail *Slist; - - slist = new(Slist); - if token == '0' { - slist.atom.integer = i; - slist.isstring = false; - } else { - slist.atom.string = new([100]char); - var i int; - for i = 0; ; i = i + 1 { - (*slist.atom.string)[i] = tokenbuf[i]; - if tokenbuf[i] == '\0' { - break; - } - } - //slist.atom.string = "hello"; // BUG! s; //= strdup(s); - slist.isstring = true; - } - slist.isatom = true; - return slist; -} - -function atoi(int <-) { // BUG: uses tokenbuf; should take argument - var v int = 0; - for i := 0; '0' <= tokenbuf[i] && tokenbuf[i] <= '9'; i = i + 1 { - v = 10 * v + convert(int, tokenbuf[i] - '0'); - } - return v; -} - -function Parse(*Slist <-) { - var slist *Slist; - - if token == EOF || token == ')' { - return nil; - } - if token == '(' { - NextToken(); - slist = ParseList(); - Expect(')'); - return slist; - } else { - // Atom - switch token { - case EOF: - return nil; - case '0': - slist = atom(atoi()); - case '"': - case 'A': - slist = atom(0); - case: - slist = nil; - print "unknown token"; //, token, tokenbuf; - } - NextToken(); - return slist; - } - return nil; -} - -function OpenFile() { - //strcpy(input, TESTSTRING); - //inputindex = 0; - // (defn foo (add 12 34))\n - inputindex = 0; - peekc = -1; // BUG - EOF = -1; // BUG - i := 0; - input[i] = '('; i = i + 1; - input[i] = 'd'; i = i + 1; - input[i] = 'e'; i = i + 1; - input[i] = 'f'; i = i + 1; - input[i] = 'n'; i = i + 1; - input[i] = ' '; i = i + 1; - input[i] = 'f'; i = i + 1; - input[i] = 'o'; i = i + 1; - input[i] = 'o'; i = i + 1; - input[i] = ' '; i = i + 1; - input[i] = '('; i = i + 1; - input[i] = 'a'; i = i + 1; - input[i] = 'd'; i = i + 1; - input[i] = 'd'; i = i + 1; - input[i] = ' '; i = i + 1; - input[i] = '1'; i = i + 1; - input[i] = '2'; i = i + 1; - input[i] = ' '; i = i + 1; - input[i] = '3'; i = i + 1; - input[i] = '4'; i = i + 1; - input[i] = ')'; i = i + 1; - input[i] = ')'; i = i + 1; - input[i] = '\n'; i = i + 1; - NextToken(); -} diff --git a/content/fosdem14.article b/content/fosdem14.article deleted file mode 100644 index eaa36755..00000000 --- a/content/fosdem14.article +++ /dev/null @@ -1,73 +0,0 @@ -Go talks at FOSDEM 2014 -24 Feb 2014 -Tags: fosdem, youtube, talk - -Andrew Gerrand - -* Introduction - -At [[http://fosdem.org/][FOSDEM]] on the 2nd of February 2014 members of the Go -community presented a series of talks in the Go Devroom. The day was a huge -success, with 13 great talks presented to a consistently jam-packed room. - -Video recordings of the talks are now available, and a selection of these -videos are presented below. - -The complete series of talks is available -[[http://www.youtube.com/playlist?list=PLtLJO5JKE5YDKG4WcaNts3IVZqhDmmuBH][as a YouTube playlist]]. -(You can also get them directly at the -[[http://video.fosdem.org/2014/K4601/Sunday/][FOSDEM video archive]].) - -* Scaling with Go: YouTube's Vitess - -Google Engineer Sugu Sougoumarane described how he and his -team built [[https://github.com/youtube/vitess][Vitess]] in Go to help scale -[[https://youtube.com][YouTube]]. - -Vitess is a set of servers and tools primarily developed in Go. -It helps scale MySQL databases for the web, and is currently used as a -fundamental component of YouTube's MySQL infrastructure. - -The talk covers some history about how and why the team chose Go, and how it -paid off. -Sugu also talks abou tips and techniques used to scale Vitess using Go. - -.iframe http://www.youtube.com/embed/qATTTSg6zXk 310 550 - -The slides for the talk are [[https://github.com/youtube/vitess/blob/master/doc/Vitess2014.pdf?raw=true][available here]]. - -* Camlistore - -[[http://camlistore.org/][Camlistore]] is designed to be "your personal storage -system for life, putting you in control, and designed to last." It's open -source, under nearly 4 years of active development, and extremely flexible. In -this talk, Brad Fitzpatrick and Mathieu Lonjaret explain why they built it, -what it does, and talk about its design. - -.iframe http://www.youtube.com/embed/yvjeIZgykiA 310 550 - -* Write your own Go compiler - -Elliot Stoneham explains the potential for Go as a portable language and -reviews the Go tools that make that such an exciting possibility. - -He said: "Based on my experiences writing an experimental Go to Haxe -translator, I'll talk about the practical issues of code generation and runtime -emulation required. I'll compare some of my design decisions with those of two -other Go compiler/translators that build on the go.tools library. My aim is to -encourage you to try one of these new 'mutant' Go compilers. I hope some of you -will be inspired to contribute to one of them or even to write a new one of -your own." - -.iframe http://www.youtube.com/embed/Qe8Dq7V3hXY 310 550 - -* More - -There were many more great talks, so please check out the complete series -[[http://www.youtube.com/playlist?list=PLtLJO5JKE5YDKG4WcaNts3IVZqhDmmuBH][as a YouTube playlist]]. -In particular, the [[http://www.youtube.com/watch?v=cwpI5ONWGxc&list=PLtLJO5JKE5YDKG4WcaNts3IVZqhDmmuBH&index=7][lightning talks]] were a lot of fun. - -I would like to give my personal thanks to the excellent speakers, Mathieu -Lonjaret for managing the video gear, and to the FOSDEM staff for making all -this possible. - diff --git a/content/from-zero-to-go-launching-on-google.article b/content/from-zero-to-go-launching-on-google.article deleted file mode 100644 index 045f6261..00000000 --- a/content/from-zero-to-go-launching-on-google.article +++ /dev/null @@ -1,204 +0,0 @@ -From zero to Go: launching on the Google homepage in 24 hours -13 Dec 2011 -Tags: appengine, google, guest - -Reinaldo Aguiar - -* Introduction - -_This_article_was_written_by_Reinaldo_Aguiar,_a_software_engineer_from_the_Search_team_at_Google._He_shares_his_experience_developing_his_first_Go_program_and_launching_it_to_an_audience_of_millions_-_all_in_one_day!_ - -I was recently given the opportunity to collaborate on a small but highly visible "20% project": the [[http://www.google.com/logos/2011/thanksgiving.html][Thanksgiving 2011 Google Doodle]]. The doodle features a turkey produced by randomly combining different styles of head, wings, feathers and legs. The user can customize it by clicking on the different parts of the turkey. This interactivity is implemented in the browser by a combination of JavaScript, CSS and of course HTML, creating turkeys on the fly. - -.image from-zero-to-go-launching-on-google_image00.png - -Once the user has created a personalized turkey it can be shared with friends and family by posting to Google+. Clicking a "Share" button (not pictured here) creates in the user's Google+ stream a post containing a snapshot of the turkey. The snapshot is a single image that matches the turkey the user created. - -With 13 alternatives for each of 8 parts of the turkey (heads, pairs of legs, distinct feathers, etc.) there are more than than 800 million possible snapshot images that could be generated. To pre-compute them all is clearly infeasible. Instead, we must generate the snapshots on the fly. Combining that problem with a need for immediate scalability and high availability, the choice of platform is obvious: Google App Engine! - -The next thing we needed to decide was which App Engine runtime to use. Image manipulation tasks are CPU-bound, so performance is the deciding factor in this case. - -To make an informed decision we ran a test. We quickly prepared a couple of equivalent demo apps for the new [[http://code.google.com/appengine/docs/python/python27/newin27.html][Python 2.7 runtime]] (which provides [[http://www.pythonware.com/products/pil/][PIL]], a C-based imaging library) and the Go runtime. Each app generates an image composed of several small images, encodes the image as a JPEG, and sends the JPEG data as the HTTP response. The Python 2.7 app served requests with a median latency of 65 milliseconds, while the Go app ran with a median latency of just 32 milliseconds. - -This problem therefore seemed the perfect opportunity to try the experimental Go runtime. - -I had no previous experience with Go and the timeline was tight: two days to be production ready. This was intimidating, but I saw it as an opportunity to test Go from a different, often overlooked angle: development velocity. How fast can a person with no Go experience pick it up and build something that performs and scales? - -* Design - -The approach was to encode the state of the turkey in the URL, drawing and encoding the snapshot on the fly. - -The base for every doodle is the background: - -.image from-zero-to-go-launching-on-google_image01.jpg - -A valid request URL might look like this: [[http://google-turkey.appspot.com/thumb/20332620][http://google-turkey.appspot.com/thumb/20332620]] - -The alphanumeric string that follows "/thumb/" indicates (in hexadecimal) which choice to draw for each layout element, as illustrated by this image: - -.image from-zero-to-go-launching-on-google_image03.png - -The program's request handler parses the URL to determine which element is selected for each component, draws the appropriate images on top of the background image, and serves the result as a JPEG. - -If an error occurs, a default image is served. There's no point serving an error page because the user will never see it - the browser is almost certainly loading this URL into an image tag. - -* Implementation - -In the package scope we declare some data structures to describe the elements of the turkey, the location of the corresponding images, and where they should be drawn on the background image. - - var ( - // dirs maps each layout element to its location on disk. - dirs = map[string]string{ - "h": "img/heads", - "b": "img/eyes_beak", - "i": "img/index_feathers", - "m": "img/middle_feathers", - "r": "img/ring_feathers", - "p": "img/pinky_feathers", - "f": "img/feet", - "w": "img/wing", - } - - // urlMap maps each URL character position to - // its corresponding layout element. - urlMap = [...]string{"b", "h", "i", "m", "r", "p", "f", "w"} - - // layoutMap maps each layout element to its position - // on the background image. - layoutMap = map[string]image.Rectangle{ - "h": {image.Pt(109, 50), image.Pt(166, 152)}, - "i": {image.Pt(136, 21), image.Pt(180, 131)}, - "m": {image.Pt(159, 7), image.Pt(201, 126)}, - "r": {image.Pt(188, 20), image.Pt(230, 125)}, - "p": {image.Pt(216, 48), image.Pt(258, 134)}, - "f": {image.Pt(155, 176), image.Pt(243, 213)}, - "w": {image.Pt(169, 118), image.Pt(250, 197)}, - "b": {image.Pt(105, 104), image.Pt(145, 148)}, - } - ) - -The geometry of the points above was calculated by measuring the actual location and size of each layout element within the image. - -Loading the images from disk on each request would be wasteful repetition, so we load all 106 images (13 * 8 elements + 1 background + 1 default) into global variables upon receipt of the first request. - - var ( - // elements maps each layout element to its images. - elements = make(map[string][]*image.RGBA) - - // backgroundImage contains the background image data. - backgroundImage *image.RGBA - - // defaultImage is the image that is served if an error occurs. - defaultImage *image.RGBA - - // loadOnce is used to call the load function only on the first request. - loadOnce [[http://golang.org/pkg/sync/#Once][sync.Once]] - ) - - // load reads the various PNG images from disk and stores them in their - // corresponding global variables. - func load() { - defaultImage = loadPNG(defaultImageFile) - backgroundImage = loadPNG(backgroundImageFile) - for dirKey, dir := range dirs { - paths, err := filepath.Glob(dir + "/*.png") - if err != nil { - panic(err) - } - for _, p := range paths { - elements[dirKey] = append(elements[dirKey], loadPNG(p)) - } - } - } - -Requests are handled in a straightforward sequence: - -- Parse the request URL, decoding the decimal value of each character in the path. - -- Make a copy of the background image as the base for the final image. - -- Draw each image element onto the background image using the layoutMap to determine where they should be drawn. - -- Encode the image as a JPEG - -- Return the image to user by writing the JPEG directly to the HTTP response writer. - -Should any error occur, we serve the defaultImage to the user and log the error to the App Engine dashboard for later analysis. - -Here's the code for the request handler with explanatory comments: - - func handler(w http.ResponseWriter, r *http.Request) { - // [[http://blog.golang.org/2010/08/defer-panic-and-recover.html][Defer]] a function to recover from any panics. - // When recovering from a panic, log the error condition to - // the App Engine dashboard and send the default image to the user. - defer func() { - if err := recover(); err != nil { - c := appengine.NewContext(r) - c.Errorf("%s", err) - c.Errorf("%s", "Traceback: %s", r.RawURL) - if defaultImage != nil { - w.Header().Set("Content-type", "image/jpeg") - jpeg.Encode(w, defaultImage, &imageQuality) - } - } - }() - - // Load images from disk on the first request. - loadOnce.Do(load) - - // Make a copy of the background to draw into. - bgRect := backgroundImage.Bounds() - m := image.NewRGBA(bgRect.Dx(), bgRect.Dy()) - draw.Draw(m, m.Bounds(), backgroundImage, image.ZP, draw.Over) - - // Process each character of the request string. - code := strings.ToLower(r.URL.Path[len(prefix):]) - for i, p := range code { - // Decode hex character p in place. - if p < 'a' { - // it's a digit - p = p - '0' - } else { - // it's a letter - p = p - 'a' + 10 - } - - t := urlMap[i] // element type by index - em := elements[t] // element images by type - if p >= len(em) { - panic(fmt.Sprintf("element index out of range %s: "+ - "%d >= %d", t, p, len(em))) - } - - // Draw the element to m, - // using the layoutMap to specify its position. - draw.Draw(m, layoutMap[t], em[p], image.ZP, draw.Over) - } - - // Encode JPEG image and write it as the response. - w.Header().Set("Content-type", "image/jpeg") - w.Header().Set("Cache-control", "public, max-age=259200") - jpeg.Encode(w, m, &imageQuality) - } - -For brevity, I've omitted several helper functions from these code listings. See the [[http://code.google.com/p/go-thanksgiving/source/browse/][source code]] for the full scoop. - -* Performance - -[[http://3.bp.blogspot.com/-3dpdQWv1nzQ/TufkFj0bS-I/AAAAAAAAAEg/hONAssh_D9c/s1600/image02.png][.image from-zero-to-go-launching-on-google_image02.png ]] - -This chart - taken directly from the App Engine dashboard - shows average request latency during launch. As you can see, even under load it never exceeds 60 ms, with a median latency of 32 milliseconds. This is wicked fast, considering that our request handler is doing image manipulation and encoding on the fly. - -* Conclusions - -I found Go's syntax to be intuitive, simple and clean. I have worked a lot with interpreted languages in the past, and although Go is instead a statically typed and compiled language, writing this app felt more like working with a dynamic, interpreted language. - -The development server provided with the [[http://code.google.com/appengine/downloads.html#Google_App_Engine_SDK_for_Go][SDK]] quickly recompiles the program after any change, so I could iterate as fast as I would with an interpreted language. It's dead simple, too - it took less than a minute to set up my development environment. - -Go's great documentation also helped me put this together fast. The docs are generated from the source code, so each function's documentation links directly to the associated source code. This not only allows the developer to understand very quickly what a particular function does but also encourages the developer to dig into the package implementation, making it easier to learn good style and conventions. - -In writing this application I used just three resources: App Engine's [[http://code.google.com/appengine/docs/go/gettingstarted/helloworld.html][Hello World Go example]], [[http://golang.org/pkg/][the Go packages documentation]], and [[http://blog.golang.org/2011/09/go-imagedraw-package.html][a blog post showcasing the Draw package]]. Thanks to the rapid iteration made possible by the development server and the language itself, I was able to pick up the language and build a super fast, production ready, doodle generator in less than 24 hours. - -Download the full app source code (including images) at [[http://code.google.com/p/go-thanksgiving/source/browse/][the Google Code project]]. - -Special thanks go to Guillermo Real and Ryan Germick who designed the doodle. diff --git a/content/from-zero-to-go-launching-on-google_image00.png b/content/from-zero-to-go-launching-on-google_image00.png deleted file mode 100644 index d84e9763..00000000 Binary files a/content/from-zero-to-go-launching-on-google_image00.png and /dev/null differ diff --git a/content/from-zero-to-go-launching-on-google_image01.jpg b/content/from-zero-to-go-launching-on-google_image01.jpg deleted file mode 100644 index a3e3e0c3..00000000 Binary files a/content/from-zero-to-go-launching-on-google_image01.jpg and /dev/null differ diff --git a/content/from-zero-to-go-launching-on-google_image02.png b/content/from-zero-to-go-launching-on-google_image02.png deleted file mode 100644 index 7bf1893b..00000000 Binary files a/content/from-zero-to-go-launching-on-google_image02.png and /dev/null differ diff --git a/content/from-zero-to-go-launching-on-google_image03.png b/content/from-zero-to-go-launching-on-google_image03.png deleted file mode 100644 index 8a92079c..00000000 Binary files a/content/from-zero-to-go-launching-on-google_image03.png and /dev/null differ diff --git a/content/gccgo-in-gcc-471.article b/content/gccgo-in-gcc-471.article deleted file mode 100644 index 4aa51c79..00000000 --- a/content/gccgo-in-gcc-471.article +++ /dev/null @@ -1,25 +0,0 @@ -Gccgo in GCC 4.7.1 -11 Jul 2012 -Tags: release - -Ian Lance Taylor - -* Introduction - -The Go language has always been defined by a [[http://golang.org/ref/spec][spec]], not an implementation. The Go team has written two different compilers that implement that spec: gc and gccgo. Having two different implementations helps ensure that the spec is complete and correct: when the compilers disagree, we fix the spec, and change one or both compilers accordingly. Gc is the original compiler, and the go tool uses it by default. Gccgo is a different implementation with a different focus, and in this post we’ll take a closer look at it. - -Gccgo is distributed as part of GCC, the GNU Compiler Collection. GCC supports several different frontends for different languages; gccgo is a Go frontend connected to the GCC backend. The Go frontend is separate from the GCC project and is designed to be able to connect to other compiler backends, but currently only supports GCC. - -Compared to gc, gccgo is slower to compile code but supports more powerful optimizations, so a CPU-bound program built by gccgo will usually run faster. All the optimizations implemented in GCC over the years are available, including inlining, loop optimizations, vectorization, instruction scheduling, and more. While it does not always produce better code, in some cases programs compiled with gccgo can run 30% faster. - -The gc compiler supports only the most popular processors: x86 (32-bit and 64-bit) and ARM. Gccgo, however, supports all the processors that GCC supports. Not all those processors have been thoroughly tested for gccgo, but many have, including x86 (32-bit and 64-bit), SPARC, MIPS, PowerPC and even Alpha. Gccgo has also been tested on operating systems that the gc compiler does not support, notably Solaris. - -Gccgo provides the standard, complete Go library. Many of the core features of the Go runtime are the same in both gccgo and gc, including the goroutine scheduler, channels, the memory allocator, and the garbage collector. Gccgo supports splitting goroutine stacks as the gc compiler does, but currently only on x86 (32-bit or 64-bit) and only when using the gold linker (on other processors, each goroutine will have a large stack, and a deep series of function calls may run past the end of the stack and crash the program). - -Gccgo distributions do not yet include a version of the go command. However, if you install the go command from a standard Go release, it already supports gccgo via the `-compiler` option: go build `-compiler`gccgo`myprog`. The tools used for calls between Go and C/C++, cgo and SWIG, also support gccgo. - -We have put the Go frontend under the same BSD license as the rest of the Go -tools. You can download the source code for the frontend at the -[[https://github.com/golang/gofrontend][gofrontend project]]. Note that when the Go frontend is linked with the GCC backend to make gccgo, GCC’s GPL license takes precedence. - -The latest release of GCC, 4.7.1, includes gccgo with support for Go 1. If you need better performance for CPU-bound Go programs, or you need to support processors or operating systems that the gc compiler does not support, gccgo might be the answer. diff --git a/content/generate.article b/content/generate.article deleted file mode 100644 index 3d4192e7..00000000 --- a/content/generate.article +++ /dev/null @@ -1,220 +0,0 @@ -Generating code -22 Dec 2014 -Tags: programming, technical - -Rob Pike - -* Generating code - -A property of universal computation—Turing completeness—is that a computer program can write a computer program. -This is a powerful idea that is not appreciated as often as it might be, even though it happens frequently. -It's a big part of the definition of a compiler, for instance. -It's also how the `go` `test` command works: it scans the packages to be tested, -writes out a Go program containing a test harness customized for the package, -and then compiles and runs it. -Modern computers are so fast this expensive-sounding sequence can complete in a fraction of a second. - -There are lots of other examples of programs that write programs. -[[http://golang.org/cmd/yacc/][Yacc]], for instance, reads in a description of a grammar and writes out a program to parse that grammar. -The protocol buffer "compiler" reads an interface description and emits structure definitions, -methods, and other support code. -Configuration tools of all sorts work like this too, examining metadata or the environment -and emitting scaffolding customized to the local state. - -Programs that write programs are therefore important elements in software engineering, -but programs like Yacc that produce source code need to be integrated into the build -process so their output can be compiled. -When an external build tool like Make is being used, this is usually easy to do. -But in Go, whose go tool gets all necessary build information from the Go source, there is a problem. -There is simply no mechanism to run Yacc from the go tool alone. - -Until now, that is. - -The [[http://blog.golang.org/go1.4][latest Go release]], 1.4, -includes a new command that makes it easier to run such tools. -It's called `go` `generate`, and it works by scanning for special comments in Go source code -that identify general commands to run. -It's important to understand that `go` `generate` is not part of `go` `build`. -It contains no dependency analysis and must be run explicitly before running `go` `build`. -It is intended to be used by the author of the Go package, not its clients. - -The `go` `generate` command is easy to use. -As a warmup, here's how to use it to generate a Yacc grammar. -Say you have a Yacc input file called `gopher.y` that defines a grammar for your new language. -To produce the Go source file implementing the grammar, -you would normally invoke the standard Go version of Yacc like this: - - go tool yacc -o gopher.go -p parser gopher.y - -The `-o` option names the output file while `-p` specifies the package name. - -To have `go` `generate` drive the process, in any one of the regular (non-generated) `.go` files -in the same directory, add this comment anywhere in the file: - - //go:generate go tool yacc -o gopher.go -p parser gopher.y - -This text is just the command above prefixed by a special comment recognized by `go` `generate`. -The comment must start at the beginning of the line and have no spaces between the `//` and the `go:generate`. -After that marker, the rest of the line specifies a command for `go` `generate` to run. - -Now run it. Change to the source directory and run `go` `generate`, then `go` `build` and so on: - - $ cd $GOPATH/myrepo/gopher - $ go generate - $ go build - $ go test - -That's it. -Assuming there are no errors, the `go` `generate` command will invoke `yacc` to create `gopher.go`, -at which point the directory holds the full set of Go source files, so we can build, test, and work normally. -Every time `gopher.y` is modified, just rerun `go` `generate` to regenerate the parser. - -For more details about how `go` `generate` works, including options, environment variables, -and so on, see the [[http://golang.org/s/go1.4-generate][design document]]. - -Go generate does nothing that couldn't be done with Make or some other build mechanism, -but it comes with the `go` tool—no extra installation required—and fits nicely into the Go ecosystem. -Just keep in mind that it is for package authors, not clients, -if only for the reason that the program it invokes might not be available on the target machine. -Also, if the containing package is intended for import by `go` `get`, -once the file is generated (and tested!) it must be checked into the -source code repository to be available to clients. - -Now that we have it, let's use it for something new. -As a very different example of how `go` `generate` can help, there is a new program available in the -`golang.org/x/tools` repository called `stringer`. -It automatically writes string methods for sets of integer constants. -It's not part of the released distribution, but it's easy to install: - - $ go get golang.org/x/tools/cmd/stringer - -Here's an example from the documentation for -[[http://godoc.org/golang.org/x/tools/cmd/stringer][`stringer`]]. -Imagine we have some code that contains a set of integer constants defining different types of pills: - - package painkiller - - type Pill int - - const ( - Placebo Pill = iota - Aspirin - Ibuprofen - Paracetamol - Acetaminophen = Paracetamol - ) - -For debugging, we'd like these constants to pretty-print themselves, which means we want a method with signature, - - func (p Pill) String() string - -It's easy to write one by hand, perhaps like this: - - func (p Pill) String() string { - switch p { - case Placebo: - return "Placebo" - case Aspirin: - return "Aspirin" - case Ibuprofen: - return "Ibuprofen" - case Paracetamol: // == Acetaminophen - return "Paracetamol" - } - return fmt.Sprintf("Pill(%d)", p) - } - -There are other ways to write this function, of course. -We could use a slice of strings indexed by Pill, or a map, or some other technique. -Whatever we do, we need to maintain it if we change the set of pills, and we need to make sure it's correct. -(The two names for paracetamol make this trickier than it might otherwise be.) -Plus the very question of which approach to take depends on the types and values: -signed or unsigned, dense or sparse, zero-based or not, and so on. - -The `stringer` program takes care of all these details. -Although it can be run in isolation, it is intended to be driven by `go` `generate`. -To use it, add a generate comment to the source, perhaps near the type definition: - - //go:generate stringer -type=Pill - -This rule specifies that `go` `generate` should run the `stringer` tool to generate a `String` method for type `Pill`. -The output is automatically written to `pill_string.go` (a default we could override with the -`-output` flag). - -Let's run it: - - $ go generate - $ cat pill_string.go - // generated by stringer -type Pill pill.go; DO NOT EDIT - - package pill - - import "fmt" - - const _Pill_name = "PlaceboAspirinIbuprofenParacetamol" - - var _Pill_index = [...]uint8{0, 7, 14, 23, 34} - - func (i Pill) String() string { - if i < 0 || i+1 >= Pill(len(_Pill_index)) { - return fmt.Sprintf("Pill(%d)", i) - } - return _Pill_name[_Pill_index[i]:_Pill_index[i+1]] - } - $ - -Every time we change the definition of `Pill` or the constants, all we need to do is run - - $ go generate - -to update the `String` method. -And of course if we've got multiple types set up this way in the same package, -that single command will update all their `String` methods with a single command. - -There's no question the generated method is ugly. -That's OK, though, because humans don't need to work on it; machine-generated code is often ugly. -It's working hard to be efficient. -All the names are smashed together into a single string, -which saves memory (only one string header for all the names, even if there are zillions of them). -Then an array, `_Pill_index`, maps from value to name by a simple, efficient technique. -Note too that `_Pill_index` is an array (not a slice; one more header eliminated) of `uint8`, -the smallest integer sufficient to span the space of values. -If there were more values, or there were negatives ones, -the generated type of `_Pill_index` might change to `uint16` or `int8`: whatever works best. - -The approach used by the methods printed by `stringer` varies according to the properties of the constant set. -For instance, if the constants are sparse, it might use a map. -Here's a trivial example based on a constant set representing powers of two: - - const _Power_name = "p0p1p2p3p4p5..." - - var _Power_map = map[Power]string{ - 1: _Power_name[0:2], - 2: _Power_name[2:4], - 4: _Power_name[4:6], - 8: _Power_name[6:8], - 16: _Power_name[8:10], - 32: _Power_name[10:12], - ..., - } - - func (i Power) String() string { - if str, ok := _Power_map[i]; ok { - return str - } - return fmt.Sprintf("Power(%d)", i) - } - - -In short, generating the method automatically allows us to do a better job than we would expect a human to do. - -There are lots of other uses of `go` `generate` already installed in the Go tree. -Examples include generating Unicode tables in the `unicode` package, -creating efficient methods for encoding and decoding arrays in `encoding/gob`, -producing time zone data in the `time` package, and so on. - -Please use `go` `generate` creatively. -It's there to encourage experimentation. - -And even if you don't, use the new `stringer` tool to write your `String` methods for your integer constants. -Let the machine do the work. diff --git a/content/getthee-to-go-meetup.article b/content/getthee-to-go-meetup.article deleted file mode 100644 index d2f586a5..00000000 --- a/content/getthee-to-go-meetup.article +++ /dev/null @@ -1,15 +0,0 @@ -Get thee to a Go meetup -27 Feb 2013 -Tags: community, talk - -Andrew Gerrand - -* Introduction - -Last week, David Symonds and I each gave talks at Sydney's Go meetup, [[http://www.meetup.com/golang-syd/][golang-syd]]. Dave spoke about [[http://talks.godoc.org/github.com/dsymonds/talks/2013-feb-golang-syd/golang-syd.slide][Go API design in the context of Protocol Buffers]], and I discussed some neat tricks in the construction of [[http://talks.godoc.org/github.com/nf/streak/talk.slide][a small command-line program]]. The presentations were short but provoked some good questions and interesting discussion. Of the 50-odd attendees, most were regular Go programmers and a fair chunk write Go code professionally. It was a fun night. - -It would have been great to see you there but, statistically, you're not from Sydney. Despair not, however, as there are likely some people in your area who either run a Go meetup or want to start one. - -The Go wiki lists [[https://golang.org/wiki/GoUserGroups][Go user groups]] around the world, so if there's one nearby you should consider going along to the next event. - -If not, why not start your own? To gauge interest, ask around in the [[https://plus.google.com/communities/114112804251407510571][Go+ Community]] and the [[http://groups.google.com/group/golang-nuts][golang-nuts mailing list]], and take a look at this [[http://golang.meetup.com/all/][list of people]] waiting for a Go meetup in their area. Once you have a few people interested - and at least one person willing to present something - pick a venue and set a date. If you build it, they will come. diff --git a/content/getting-to-know-go-community.article b/content/getting-to-know-go-community.article deleted file mode 100644 index 9b51ab31..00000000 --- a/content/getting-to-know-go-community.article +++ /dev/null @@ -1,17 +0,0 @@ -Getting to know the Go community -21 Dec 2011 -Tags: community - -Andrew Gerrand - -* Introduction - -Over the past couple of years Go has attracted a lot of users and contributors, and I've had a great time meeting and talking with many of you. However, for every Gopher I know there are dozens I know nothing about. In order to address this imbalance I've prepared a survey for Go users everywhere. - -The survey is short. It asks about you, your involvement with Go, and and your interest in Go-related events. Among other things, this data will help myself and the rest of the Go team plan future Go events and schedule conference appearances. - -Please take a minute to [[https://docs.google.com/spreadsheet/viewform?hl=en_US&formkey=dFdxOTU1Mm92VFA3eHdkekFPamJjUnc6MQ#gid=0][complete the survey now]]. - -Thanks! - -Andrew diff --git a/content/gif-decoder-exercise-in-go-interfaces.article b/content/gif-decoder-exercise-in-go-interfaces.article deleted file mode 100644 index 3496ea60..00000000 --- a/content/gif-decoder-exercise-in-go-interfaces.article +++ /dev/null @@ -1,120 +0,0 @@ -A GIF decoder: an exercise in Go interfaces -25 May 2011 -Tags: gif, gopher, image, interface, lagomorph, lzw, moustache, rodent, technical - -Rob Pike - -* Introduction - -At the Google I/O conference in San Francisco on May 10, 2011, we announced that the Go language is now available on Google App Engine. Go is the first language to be made available on App Engine that compiles directly to machine code, which makes it a good choice for CPU-intensive tasks such as image manipulation. - -In that vein, we demonstrated a program called [[http://moustach-io.appspot.com/][Moustachio]] that makes it easy to improve a picture such as this one: - -.image gif-decoder-exercise-in-go-interfaces_image00.jpg - -by adding a moustache and sharing the result: - -.image gif-decoder-exercise-in-go-interfaces_image02.jpg - -All the graphical processing, including rendering the antialiased moustache, is done by a Go program running on App Engine. (The source is available at [[http://code.google.com/p/appengine-go/source/browse/example/moustachio/][the appengine-go project]].) - -Although most images on the web—at least those likely to be moustachioed—are JPEGs, there are countless other formats floating around, and it seemed reasonable for Moustachio to accept uploaded images in a few of them. JPEG and PNG decoders already existed in the Go image library, but the venerable GIF format was not represented, so we decided to write a GIF decoder in time for the announcement. That decoder contains a few pieces that demonstrate how Go's interfaces make some problems easier to solve. The rest of this blog post describes a couple of instances. - -* The GIF format - -First, a quick tour of the GIF format. A GIF image file is _paletted_, that is, each pixel value is an index into a fixed color map that is included in the file. The GIF format dates from a time when there were usually no more than 8 bits per pixel on the display, and a color map was used to convert the limited set of values into the RGB (red, green, blue) triples needed to light the screen. (This is in contrast to a JPEG, for example, which has no color map because the encoding represents the distinct color signals separately.) - -A GIF image can contain anywhere from 1 to 8 bits per pixel, inclusive, but 8 bits per pixel is the most common. - -Simplifying somewhat, a GIF file contains a header defining the pixel depth and image dimensions, a color map (256 RGB triples for an 8-bit image), and then the pixel data. The pixel data is stored as a one-dimensional bit stream, compressed using the LZW algorithm, which is quite effective for computer-generated graphics although not so good for photographic imagery. The compressed data is then broken into length-delimited blocks with a one-byte count (0-255) followed by that many bytes: - -.image gif-decoder-exercise-in-go-interfaces_image03.gif - -* Deblocking the pixel data - -To decode GIF pixel data in Go, we can use the LZW decompressor from the `compress/lzw` package. It has a NewReader function that returns an object that, as [[http://golang.org/pkg/compress/lzw/#NewReader][the documentation]] says, "satisfies reads by decompressing the data read from r": - - func NewReader(r io.Reader, order Order, litWidth int) io.ReadCloser - -Here `order` defines the bit-packing order and `litWidth` is the word size in bits, which for a GIF file corresponds to the pixel depth, typically 8. - -But we can't just give `NewReader` the input file as its first argument because the decompressor needs a stream of bytes but the GIF data is a stream of blocks that must be unpacked. To address this problem, we can wrap the input `io.Reader` with some code to deblock it, and make that code again implement `Reader`. In other words, we put the deblocking code into the `Read` method of a new type, which we call `blockReader`. - -Here's the data structure for a `blockReader`. - - type blockReader struct { - r reader // Input source; implements io.Reader and io.ByteReader. - slice []byte // Buffer of unread data. - tmp [256]byte // Storage for slice. - } - -The reader, `r`, will be the source of the image data, perhaps a file or HTTP connection. The `slice` and `tmp` fields will be used to manage the deblocking. Here's the `Read` method in its entirety. It's a nice example of the use of slices and arrays in Go. - - 1 func (b *blockReader) Read(p []byte) (int, os.Error) { - 2 if len(p) == 0 { - 3 return 0, nil - 4 } - 5 if len(b.slice) == 0 { - 6 blockLen, err := b.r.ReadByte() - 7 if err != nil { - 8 return 0, err - 9 } - 10 if blockLen == 0 { - 11 return 0, os.EOF - 12 } - 13 b.slice = b.tmp[0:blockLen] - 14 if _, err = io.ReadFull(b.r, b.slice); err != nil { - 15 return 0, err - 16 } - 17 } - 18 n := copy(p, b.slice) - 19 b.slice = b.slice[n:] - 20 return n, nil - 21 } - -Lines 2-4 are just a sanity check: if there's no place to put data, return zero. That should never happen, but it's good to be safe. - -Line 5 asks if there's data left over from a previous call by checking the length of `b.slice`. If there isn't, the slice will have length zero and we need to read the next block from `r`. - -A GIF block starts with a byte count, read on line 6. If the count is zero, GIF defines this to be a terminating block, so we return `EOF` on line 11. - -Now we know we should read `blockLen` bytes, so we point `b.slice` to the first `blockLen` bytes of `b.tmp` and then use the helper function `io.ReadFull` to read that many bytes. That function will return an error if it can't read exactly that many bytes, which should never happen. Otherwise we have `blockLen` bytes ready to read. - -Lines 18-19 copy the data from `b.slice` to the caller's buffer. We are implementing `Read`, not `ReadFull`, so we are allowed to return fewer than the requested number of bytes. That makes it easy: we just copy the data from `b.slice` to the caller's buffer (`p`), and the return value from copy is the number of bytes transferred. Then we reslice `b.slice` to drop the first `n` bytes, ready for the next call. - -It's a nice technique in Go programming to couple a slice (`b.slice`) to an array (`b.tmp`). In this case, it means `blockReader` type's `Read` method never does any allocations. It also means we don't need to keep a count around (it's implicit in the slice length), and the built-in `copy` function guarantees we never copy more than we should. (For more about slices, see [[http://blog.golang.org/2011/01/go-slices-usage-and-internals.html][this post from the Go Blog]].) - -Given the `blockReader` type, we can unblock the image data stream just by wrapping the input reader, say a file, like this: - - deblockingReader := &blockReader{r: imageFile} - -This wrapping turns a block-delimited GIF image stream into a simple stream of bytes accessible by calls to the `Read` method of the `blockReader`. - -* Connecting the pieces - -With `blockReader` implemented and the LZW compressor available from the library, we have all the pieces we need to decode the image data stream. We stitch them together with this thunderclap, straight from the code: - - lzwr := lzw.NewReader(&blockReader{r: d.r}, lzw.LSB, int(litWidth)) - if _, err = io.ReadFull(lzwr, m.Pix); err != nil { - break - } - -That's it. - -The first line creates a `blockReader` and passes it to `lzw.NewReader` to create a decompressor. Here `d.r` is the `io.Reader` holding the image data, `lzw.LSB` defines the byte order in the LZW decompressor, and `litWidth` is the pixel depth. - -Given the decompressor, the second line calls `io.ReadFull` to decompress the data and store it in the image, `m.Pix`. When `ReadFull` returns, the image data is decompressed and stored in the image, `m`, ready to be displayed. - -This code worked first time. Really. - -We could avoid the temporary variable `lzwr` by placing the `NewReader` call into the argument list for `ReadFull`, just as we built the `blockReader` inside the call to `NewReader`, but that might be packing too much into a single line of code. - -* Conclusion - -Go's interfaces make it easy to construct software by assembling piece parts like this to restructure data. In this example, we implemented GIF decoding by chaining together a deblocker and a decompressor using the `io.Reader` interface, analogous to a type-safe Unix pipeline. Also, we wrote the deblocker as an (implicit) implementation of a `Reader` interface, which then required no extra declaration or boilerplate to fit it into the processing pipeline. It's hard to implement this decoder so compactly yet cleanly and safely in most languages, but the interface mechanism plus a few conventions make it almost natural in Go. - -That deserves another picture, a GIF this time: - -.image gif-decoder-exercise-in-go-interfaces_image01.gif - -The GIF format is defined at [[http://www.w3.org/Graphics/GIF/spec-gif89a.txt][http://www.w3.org/Graphics/GIF/spec-gif89a.txt]]. diff --git a/content/gif-decoder-exercise-in-go-interfaces_image00.jpg b/content/gif-decoder-exercise-in-go-interfaces_image00.jpg deleted file mode 100644 index 7301e460..00000000 Binary files a/content/gif-decoder-exercise-in-go-interfaces_image00.jpg and /dev/null differ diff --git a/content/gif-decoder-exercise-in-go-interfaces_image01.gif b/content/gif-decoder-exercise-in-go-interfaces_image01.gif deleted file mode 100644 index 060544a3..00000000 Binary files a/content/gif-decoder-exercise-in-go-interfaces_image01.gif and /dev/null differ diff --git a/content/gif-decoder-exercise-in-go-interfaces_image02.jpg b/content/gif-decoder-exercise-in-go-interfaces_image02.jpg deleted file mode 100644 index e5122dfe..00000000 Binary files a/content/gif-decoder-exercise-in-go-interfaces_image02.jpg and /dev/null differ diff --git a/content/gif-decoder-exercise-in-go-interfaces_image03.gif b/content/gif-decoder-exercise-in-go-interfaces_image03.gif deleted file mode 100644 index 2d6321bc..00000000 Binary files a/content/gif-decoder-exercise-in-go-interfaces_image03.gif and /dev/null differ diff --git a/content/go-11-is-released.article b/content/go-11-is-released.article deleted file mode 100644 index 1245c30e..00000000 --- a/content/go-11-is-released.article +++ /dev/null @@ -1,31 +0,0 @@ -Go 1.1 is released -13 May 2013 -Tags: release - -Andrew Gerrand - -* Introduction - -It is our great pleasure to announce the release of Go 1.1. - -.image go-11-is-released_gopherbiplane5.jpg - -In March last year we released Go 1.0, and since then we have released three minor "point releases". The point releases were made to fix only critical issues, so the Go 1.0.3 you use today is still, in essence, the Go 1.0 we released in March 2012. - -Go 1.1 includes many improvements over 1.0. - -The most significant improvements are performance-related. We have made optimizations in the compiler and linker, garbage collector, goroutine scheduler, map implementation, and parts of the standard library. It is likely that your Go code will run noticeably faster when built with Go 1.1. - -There are some minor changes to the language itself, two of which are worth singling out here: the [[http://golang.org/doc/go1.1#return][changes to return requirements]] will lead to more succinct and correct programs, and the introduction of [[http://golang.org/doc/go1.1#method_values][method values]] provides an expressive way to bind a method to its receiver as a function value. - -Concurrent programming is safer in Go 1.1 with the addition of a race detector for finding memory synchronization errors in your programs. We will discuss the race detector more in an upcoming article, but for now [[http://golang.org/doc/articles/race_detector.html][the manual]] is a great place to get started. - -The tools and standard library have been improved and expanded. You can read the full story in the [[http://golang.org/doc/go1.1][release notes]]. - -As per our [[http://golang.org/doc/go1compat.html][compatibility guidelines]], Go 1.1 remains compatible with Go 1.0 and we recommend all Go users upgrade to the new release. - -All this would not have been possible without the help of our contributors from the open source community. Since Go 1.0, the core received more than 2600 commits from 161 people outside Google. Thank you everyone for your time and effort. In particular, we would like to thank Shenghou Ma, Rémy Oudompheng, Dave Cheney, Mikio Hara, Alex Brainman, Jan Ziak, and Daniel Morsing for their outstanding contributions. - -To grab the new release, follow the usual [[http://golang.org/doc/install][installation instructions]]. Happy hacking! - -_Thanks_to_Renée_French_for_the_gopher!_ diff --git a/content/go-11-is-released_gopherbiplane5.jpg b/content/go-11-is-released_gopherbiplane5.jpg deleted file mode 100644 index 9b56e23c..00000000 Binary files a/content/go-11-is-released_gopherbiplane5.jpg and /dev/null differ diff --git a/content/go-and-google-app-engine.article b/content/go-and-google-app-engine.article deleted file mode 100644 index e12feb4a..00000000 --- a/content/go-and-google-app-engine.article +++ /dev/null @@ -1,29 +0,0 @@ -Go and Google App Engine -10 May 2011 -Tags: appengine, release - -David Symonds - -Nigel Tao - -Andrew Gerrand - -* Introduction - -Google’s App Engine provides a reliable, scalable, easy way to build and deploy applications for the web. Over a hundred thousand apps are hosted at appspot.com and custom domains using the App Engine infrastructure. Originally written for Python apps, in 2009 the system added a Java runtime. And today, at Google I/O, we’re thrilled to announce that Go will be next. It’s marked as an experimental App Engine feature for now, because it’s early days, but both the App Engine and Go teams are very excited about this milestone. - -By early days, we mean that it’s still rolling out. As of today, the App Engine SDK for Go is [[http://code.google.com/p/googleappengine/downloads/list][available for download]], and we will soon enable deployment of Go apps into the App Engine hosting infrastructure. Today, through the SDK, you’ll be able to write web apps, learn about the APIs (and the language, if it’s new to you), and run your web app locally. Once full deployment is enabled, it’ll be easy to push your app to Google’s cloud. - -One of the cool but less obvious things about this news is that it provides a very easy way to play with Go. You don’t even need to have Go installed beforehand because the SDK is fully self-contained. Just download the SDK, unzip it, and start coding. Moreover, the SDK’s “dev app server” means you don’t even need to run the compiler yourself; everything is delightfully automatic. - -What you’ll find in the SDK is many of the standard App Engine APIs, custom designed in good Go style, including Datastore, Blobstore, URL Fetch, Mail, Users, and so on. More APIs will be added as the environment develops. The runtime provides the full Go language and almost all the standard libraries, except for a few things that don’t make sense in the App Engine environment. For instance, there is no `unsafe` package and the `syscall` package is trimmed. (The implementation uses an expanded version of the setup in the [[http://golang.org/doc/play/][Go Playground]] on [[http://golang.org/][golang.org]].) - -Also, although goroutines and channels are present, when a Go app runs on App Engine only one thread is run in a given instance. That is, all goroutines run in a single operating system thread, so there is no CPU parallelism available for a given client request. We expect this restriction will be lifted at some point. - -Despite these minor restrictions, it’s the real language: Code is deployed in source form and compiled in the cloud using the 64-bit x86 compiler (6g), making it the first true compiled language that runs on App Engine. Go on App Engine makes it possible to deploy efficient, CPU-intensive web applications. - -If you want to know more, read the [[http://code.google.com/appengine/docs/go/][documentation]] (start with “[[http://code.google.com/appengine/docs/go/gettingstarted/][Getting Started]]”). The libraries and SDK are open source, hosted at [[http://code.google.com/p/appengine-go/][http://code.google.com/p/appengine-go/]]. We’ve created a new [[http://groups.google.com/group/google-appengine-go][google-appengine-go]] mailing list; feel free to contact us there with App Engine-specific questions. The [[http://code.google.com/p/googleappengine/issues/list][issue tracker for App Engine]] is the place for reporting issues related to the new Go SDK. - -The Go App Engine SDK is [[http://code.google.com/p/googleappengine/downloads/list][available]] for Linux and Mac OS X (10.5 or greater); we hope a Windows version will also be available soon. - -We’d like to offer our thanks for all the help and enthusiasm we received from Google’s App Engine team in making this happen. diff --git a/content/go-and-google-cloud-platform.article b/content/go-and-google-cloud-platform.article deleted file mode 100644 index a7f54f2f..00000000 --- a/content/go-and-google-cloud-platform.article +++ /dev/null @@ -1,36 +0,0 @@ -Go and the Google Cloud Platform -12 Jun 2013 - -Andrew Gerrand - -* Introduction - -In 2011 we announced the [[https://developers.google.com/appengine/docs/go/overview][Go runtime for App Engine]]. Since then, we have continued to improve the Go App Engine experience, and generally improved Go support for the Google Cloud Platform. For instance, the [[http://code.google.com/p/google-api-go-client][google-api-go-client]] provides a Go interface to a range of Google's public APis, including Compute Engine, Cloud Storage, BigQuery, Drive, and many more. - -Learn more by watching these talks from Google I/O this year: - -* High Performance Apps with Go on App Engine - -_The_Go_runtime_for_App_Engine_is_a_high_performance_engine_for_ -_running_web_applications._It_produces_fast_responses,_ -_starts_instances_in_a_fraction_of_a_second,_makes_the_most_use_ -_of_instance_hours,_and_allows_your_app_to_do_serious_processing_ -_at_full_machine_speed._ -_Come_along_to_hear_how_to_fully_exploit_the_power_of_Go_on_App_ -_Engine_and_make_your_web_applications_the_best_they_can_be._ - -.iframe http://www.youtube.com/embed/fc25ihfXhbg 309 549 - -* All the Ships in the World - -Visualizing Data with Google Cloud and Maps - -_Tens_of_thousands_of_ships_report_their_position_at_least_once_ -_every_5_minutes,_24_hours_a_day._ -_Visualizing_that_quantity_of_data_and_serving_it_out_to_large_ -_numbers_of_people_takes_lots_of_power_both_in_the_browser_and_on_the_server._ -_This_session_will_explore_the_use_of_Maps,_ -_App_Engine,_Go,_Compute_Engine,_BigQuery,_Cloud_Storage,_ -_and_WebGL_to_do_massive_data_visualization._ - -.iframe http://www.youtube.com/embed/MT7cd4M9vzs 309 549 diff --git a/content/go-app-engine-sdk-155-released.article b/content/go-app-engine-sdk-155-released.article deleted file mode 100644 index 0249ba0c..00000000 --- a/content/go-app-engine-sdk-155-released.article +++ /dev/null @@ -1,25 +0,0 @@ -Go App Engine SDK 1.5.5 released -11 Oct 2011 -Tags: appengine, gofix, release - -Andrew Gerrand - -* Introduction - -Today we released version 1.5.5 the Go App Engine SDK. You can download it from the [[http://code.google.com/appengine/downloads.html][App Engine downloads page]]. - -This release includes changes and improvements to the App Engine APIs and brings the supporting Go tool chain to [[http://golang.org/doc/devel/release.html#r60][release.r60.2]] (the current stable release). Also included in this release are the [[http://golang.org/cmd/godoc/][godoc]], [[http://golang.org/cmd/gofmt/][gofmt]], and [[http://golang.org/cmd/gofix/][gofix]] tools from the Go tool chain. They can be found in the root directory of the SDK. - -Some changes made in this release are backwards-incompatible, so we have incremented the SDK `api_version` to 3. Existing apps will require code changes when migrating to `api_version` 3. - -The gofix tool that ships with the SDK has been customized with App Engine-specific modules. It can be used to automatically update Go apps to work with the latest appengine packages and the updated Go standard library. To update your apps, run: - - /path/to/sdk/gofix /path/to/your/app - -The SDK now includes the appengine package source code, so you can use the local godoc to read App Engine API documentation: - - /path/to/sdk/godoc appengine/datastore Get - -*Important*note:* We have deprecated `api_version` 2. Go apps that use `api_version` 2 will stop working after 16 December 2011. Please update your apps to use api_version 3 before then. - -See the [[http://code.google.com/p/googleappengine/wiki/SdkForGoReleaseNotes][release notes]] for a full list of changes. Please direct any questions about the new SDK to the [[http://groups.google.com/group/google-appengine-go][Go App Engine discussion group]]. diff --git a/content/go-at-google-io-2011-videos.article b/content/go-at-google-io-2011-videos.article deleted file mode 100644 index bc2aacdf..00000000 --- a/content/go-at-google-io-2011-videos.article +++ /dev/null @@ -1,42 +0,0 @@ -Go at Google I/O 2011: videos -23 May 2011 -Tags: appengine, gopher, talk, video - -Andrew Gerrand - -* Introduction - -The Go team had a great time at Google I/O 2011. It was a pleasure to meet so many programmers who share our enthusiasm for Go, and to share our work of the past few months. For those of you that couldn’t be there in person, you can now watch videos of our two Go presentations on YouTube. - -* Writing Web Apps in Go - -In “[[http://www.youtube.com/watch?v=-i0hat7pdpk][Writing Web Apps in Go]]” we announce the [[http://blog.golang.org/2011/05/go-and-google-app-engine.html][Go runtime for Google App Engine]] and walk through the development and deployment of [[http://moustach-io.appspot.com/][Moustachio]], the first Go App Engine app. - -.iframe http://www.youtube.com/embed/-i0hat7pdpk 349 560 - -(See the [[http://golang.org/doc/talks/io2011/Writing_Web_Apps_in_Go.pdf][presentation slides]].) - -The [[https://code.google.com/p/appengine-go/source/browse/example/moustachio][source code]] for Moustachio is available as part of [[http://code.google.com/appengine/downloads.html#Google_App_Engine_SDK_for_Go][the SDK]] along with some other examples, such as this [[http://mandelbrot-tiles.appspot.com/][Mandelbrot demo]]. - -Most important, this talk features the debut of the plush gopher. - -.image go-at-google-io-2011-videos_gopher.jpg - -For those that didn’t get one at the conference, we hope to make him available for purchase online soon. - -* Real World Go - -“[[http://www.youtube.com/watch?v=7QDVRowyUQA][Real World Go]]”, presented at [[http://io-bootcamp.com][I/O Bootcamp]], gives a brief introduction to Go and four case studies of its use in solving real problems: -- [[http://heroku.com][Heroku]] with [[https://github.com/ha/doozerd][Doozer]], a highly available consistent data store, - -- [[http://mroffice.org/telephony.html][MROffice Dialer]], a VOIP system for call centers, - -- [[http://www.atlassian.com][Atlassian]]’s virtual machine cluster management system, - -- [[http://www.camlistore.org][Camlistore]], a content addressable storage system. - -.iframe http://www.youtube.com/embed/7QDVRowyUQA 349 560 - -(See the [[http://golang.org/doc/talks/io2011/Real_World_Go.pdf ][presentation slides]].) - -Thanks to everyone who attended our talks and workshops. We look forward to seeing you again soon! diff --git a/content/go-at-google-io-2011-videos_gopher.jpg b/content/go-at-google-io-2011-videos_gopher.jpg deleted file mode 100644 index aff98d44..00000000 Binary files a/content/go-at-google-io-2011-videos_gopher.jpg and /dev/null differ diff --git a/content/go-at-heroku.article b/content/go-at-heroku.article deleted file mode 100644 index 8f0892d3..00000000 --- a/content/go-at-heroku.article +++ /dev/null @@ -1,27 +0,0 @@ -Go at Heroku -21 Apr 2011 -Tags: guest - -Keith Rarick - -Blake Mizerany - -* Introduction - -_This_week’s_blog_post_is_written_by_ [[http://xph.us/][_Keith_Rarick_]] _and_ [[http://itsbonus.heroku.com/][_Blake_Mizerany_]], _systems_engineers_at_ [[http://www.heroku.com/][Heroku]]. _In_their_own_words,_they_"eat,_drink,_and_sleep_distributed_systems."_Here_they_discuss_their_experiences_using_Go._ - -A big problem that comes with building distributed systems is the coordination of physical servers. Each server needs to know various facts about the system as a whole. This critical data includes locks, configuration data, and so on, and it must be consistent and available even during data store failures, so we need a data store with solid consistency guarantees. Our solution to this problem is [[http://xph.us/2011/04/13/introducing-doozer.html][Doozer]], a new, consistent, highly-available data store written in Go. - -At Doozer's core is [[http://en.wikipedia.org/wiki/Paxos_(computer_science)][Paxos]], a family of protocols for solving consensus in an unreliable network of unreliable nodes. While Paxos is essential to running a fault-tolerant system, it is notorious for being difficult to implement. Even example implementations that can be found online are complex and hard to follow, despite being simplified for educational purposes. Existing production systems have a reputation for being worse. - -Fortunately, Go's concurrency primitives made the task much easier. Paxos is defined in terms of independent, concurrent processes that communicate via passing messages. In Doozer, these processes are implemented as goroutines, and their communications as channel operations. In the same way that garbage collectors improve upon malloc and free, we found that [[http://blog.golang.org/2010/07/share-memory-by-communicating.html][goroutines and channels]] improve upon the lock-based approach to concurrency. These tools let us avoid complex bookkeeping and stay focused on the problem at hand. We are still amazed at how few lines of code it took to achieve something renowned for being difficult. - -The standard packages in Go were another big win for Doozer. The Go team is very pragmatic about what goes into them. For instance, a package we quickly found useful was [[http://golang.org/pkg/websocket/][websocket]]. Once we had a working data store, we needed an easy way to introspect it and visualize activity. Using the websocket package, Keith was able to add the web viewer on his train ride home and without requiring external dependencies. This is a real testament to how well Go mixes systems and application programming. - -One of our favorite productivity gains was provided by Go's source formatter: [[http://golang.org/cmd/gofmt/][gofmt]]. We never argued over where to put a curly-brace, tabs vs. spaces, or if we should align assignments. We simply agreed that the buck stopped at the default output from gofmt. - -Deploying Doozer was satisfyingly simple. Go builds statically linked binaries which means Doozer has no external dependencies; it's a single file that can be copied to any machine and immediately launched to join a cluster of running Doozers. - -Finally, Go's maniacal focus on simplicity and orthogonality aligns with our view of software engineering. Like the Go team, we are pragmatic about what features go into Doozer. We sweat the details, preferring to change an existing feature instead of introducing a new one. In this sense, Go is a perfect match for Doozer. - -We already have future projects in mind for Go. Doozer is just the start of much bigger system. diff --git a/content/go-at-io-frequently-asked-questions.article b/content/go-at-io-frequently-asked-questions.article deleted file mode 100644 index fb36f844..00000000 --- a/content/go-at-io-frequently-asked-questions.article +++ /dev/null @@ -1,33 +0,0 @@ -Go at I/O: Frequently Asked Questions -27 May 2010 -Tags: appengine - -Andrew Gerrand - -* Introduction - -Among the high-profile product launches at Google I/O last week, our small team gave presentations to packed rooms and met many present and future Go programmers. It was especially gratifying to meet with so many people who, after learning a bit about Go, were excited by the potential benefits (both immediate and long-term) they could gain from using it. - -We were asked a lot of good questions during I/O, and in this post I'd like to recap and expand upon some of them. - -How suitable is Go for production systems? -Go is ready and stable now. We are pleased to report that Google is using Go for some production systems, and they are performing well. Of course there is still room for improvement - that's why we're continuing to work on the language, libraries, tools, and runtime. - -Do you have plans to implement generics? -Many proposals for generics-like features have been mooted both publicly and internally, but as yet we haven't found a proposal that is consistent with the rest of the language. We think that one of Go's key strengths is its simplicity, so we are wary of introducing new features that might make the language more difficult to understand. Additionally, the more Go code we write (and thus the better we learn how to write Go code ourselves), the less we feel the need for such a language feature. - -Do you have any plans to support GPU programming? -We don't have any immediate plans to do this, but as Go is architecture-agnostic it's quite possible. The ability to launch a goroutine that runs on a different processor architecture, and to use channels to communicate between goroutines running on separate architectures, seem like good ideas. - -Are there plans to support Go under App Engine? -Both the Go and App Engine teams would like to see this happen. As always, it is a question of resources and priorities as to if and when it will become a reality. - -Are there plans to support Go under Android? -Both Go compilers support ARM code generation, so it is possible. While we think Go would be a great language for writing mobile applications, Android support is not something that's being actively worked on. - -What can I use Go for? -Go was designed with systems programming in mind. Servers, clients, databases, caches, balancers, distributors - these are applications Go is obviously useful for, and this is how we have begun to use it within Google. However, since Go's open-source release, the community has found a diverse range of applications for the language. From web apps to games to graphics tools, Go promises to shine as a general-purpose programming language. The potential is only limited by library support, which is improving at a tremendous rate. Additionally, educators have expressed interest in using Go to teach programming, citing its succinct syntax and consistency as well-suited to the task. - -Thanks to everyone who attended our presentations, or came to talk with us at Office Hours. We hope to see you again at future events. - -The video of [[http://code.google.com/events/io/2010/sessions/go-programming.html][Rob and Russ' talk]] will be uploaded to YouTube within the next week, and will then be posted on this blog. diff --git a/content/go-becomes-more-stable.article b/content/go-becomes-more-stable.article deleted file mode 100644 index e33d2c51..00000000 --- a/content/go-becomes-more-stable.article +++ /dev/null @@ -1,21 +0,0 @@ -Go becomes more stable -16 Mar 2011 -Tags: release - -Andrew Gerrand - -* Introduction - -The Go project is moving fast. As we learn more about Go we are compelled to change our tools, libraries, and occasionally even the language itself. We permit backward-incompatible changes so that we can learn from, rather than immortalize, our mistakes. We believe flexibility at this stage of Go’s development is essential to the project’s development and, ultimately, its longevity. - -Since Go's launch we have been making releases approximately once a week. Accompanying each release are [[http://golang.org/doc/devel/release.html][notes describing what changed]], with any backwards-incompatible changes signposted. Questions I hear often are "Is Go stable? How can I be sure that I won’t have to update my Go code every week?" The answer to those questions are now "Yes," and "You won't." - -With this week’s release we’re introducing a new release tagging scheme. We intend to continue with our weekly releases, but have renamed the existing tags from `release` to `weekly`. The `release` tag will now be applied to one hand-picked stable release each month or two. This more relaxed release schedule should make life easier for the average Go programmer. - -Users will still need to update their code periodically (this is the cost of using a young language) but with less frequency. An additional benefit is that by tagging stable releases less often we can put more effort into automating updates. To this end we have introduced gofix, a tool that will help you update your code. - -The revision formerly tagged `release.2011-03-07.1` (now `weekly.2011-03-07.1`) has been nominated our first stable release, and has been given the tag `release.r56`. As we tag each stable release we will post an announcement to the new [[http://groups.google.com/group/golang-announce][golang-announce]] mailing list. (Why not [[http://groups.google.com/group/golang-announce/subscribe][subscribe now]]?) - -What’s the upshot of all this? You can continue to keep your Go installation updated using `hg`update`release`, but now you should only need to update when we tag a new stable release. If you wish to stay at the leading edge, you should switch to the weekly tag with `hg`update`weekly`. - -Happy coding! diff --git a/content/go-concurrency-patterns-timing-out-and.article b/content/go-concurrency-patterns-timing-out-and.article deleted file mode 100644 index 9d60ead0..00000000 --- a/content/go-concurrency-patterns-timing-out-and.article +++ /dev/null @@ -1,51 +0,0 @@ -Go Concurrency Patterns: Timing out, moving on -23 Sep 2010 -Tags: concurrency, technical - -Andrew Gerrand - -* Introduction - -Concurrent programming has its own idioms. A good example is timeouts. Although Go's channels do not support them directly, they are easy to implement. Say we want to receive from the channel `ch`, but want to wait at most one second for the value to arrive. We would start by creating a signalling channel and launching a goroutine that sleeps before sending on the channel: - - timeout := make(chan bool, 1) - go func() { - time.Sleep(1 * time.Second) - timeout <- true - }() - -We can then use a `select` statement to receive from either `ch` or `timeout`. If nothing arrives on `ch` after one second, the timeout case is selected and the attempt to read from ch is abandoned. - - select { - case <-ch: - // a read from ch has occurred - case <-timeout: - // the read from ch has timed out - } - -The `timeout` channel is buffered with space for 1 value, allowing the timeout goroutine to send to the channel and then exit. The goroutine doesn't know (or care) whether the value is received. This means the goroutine won't hang around forever if the `ch` receive happens before the timeout is reached. The `timeout` channel will eventually be deallocated by the garbage collector. - -(In this example we used `time.Sleep` to demonstrate the mechanics of goroutines and channels. In real programs you should use ` [[http://golang.org/pkg/time/#After][time.After]]`, a function that returns a channel and sends on that channel after the specified duration.) - -Let's look at another variation of this pattern. In this example we have a program that reads from multiple replicated databases simultaneously. The program needs only one of the answers, and it should accept the answer that arrives first. - -The function `Query` takes a slice of database connections and a `query` string. It queries each of the databases in parallel and returns the first response it receives: - - func Query(conns []Conn, query string) Result { - ch := make(chan Result, 1) - for _, conn := range conns { - go func(c Conn) { - select { - case ch <- c.DoQuery(query): - default: - } - }(conn) - } - return <-ch - } - -In this example, the closure does a non-blocking send, which it achieves by using the send operation in `select` statement with a `default` case. If the send cannot go through immediately the default case will be selected. Making the send non-blocking guarantees that none of the goroutines launched in the loop will hang around. However, if the result arrives before the main function has made it to the receive, the send could fail since no one is ready. - -This problem is a textbook example of what is known as a [[https://en.wikipedia.org/wiki/Race_condition][race condition]], but the fix is trivial. We just make sure to buffer the channel `ch` (by adding the buffer length as the second argument to [[http://golang.org/pkg/builtin/#make][make]]), guaranteeing that the first send has a place to put the value. This ensures the send will always succeed, and the first value to arrive will be retrieved regardless of the order of execution. - -These two examples demonstrate the simplicity with which Go can express complex interactions between goroutines. diff --git a/content/go-fmt-your-code.article b/content/go-fmt-your-code.article deleted file mode 100644 index 86f7dc12..00000000 --- a/content/go-fmt-your-code.article +++ /dev/null @@ -1,62 +0,0 @@ -go fmt your code -23 Jan 2013 -Tags: gofix, gofmt, technical - -Andrew Gerrand - -* Introduction - -[[http://golang.org/cmd/gofmt/][Gofmt]] is a tool that automatically formats Go source code. - -Gofmt'd code is: - -- easier to *write*: never worry about minor formatting concerns while hacking away, - -- easier to *read*: when all code looks the same you need not mentally convert others' formatting style into something you can understand. - -- easier to *maintain*: mechanical changes to the source don't cause unrelated changes to the file's formatting; diffs show only the real changes. - -- *uncontroversial*: never have a debate about spacing or brace position ever again! - -* Format your code - -We recently conducted a survey of Go packages in the wild and found that about 70% of them are formatted according to gofmt's rules. This was more than expected - and thanks to everyone who uses gofmt - but it would be great to close the gap. - -To format your code, you can use the gofmt tool directly: - - gofmt -w yourcode.go - -Or you can use the "[[http://golang.org/cmd/go/#hdr-Run_gofmt_on_package_sources][go fmt]]" command: - - go fmt path/to/your/package - -To help keep your code in the canonical style, the Go repository contains hooks for editors and version control systems that make it easy to run gofmt on your code. - -For Vim users, the [[http://tip.golang.org/misc/vim/readme.txt][Vim plugin for Go]] includes the [[http://tip.golang.org/misc/vim/ftplugin/go/fmt.vim][:Fmt command]] that runs gofmt on the current buffer. - -For emacs users, [[http://tip.golang.org/misc/emacs/go-mode.el][go-mode.el]] provides a gofmt-before-save hook that can be installed by adding this line to your .emacs file: - - (add-hook 'before-save-hook #'gofmt-before-save) - -For Eclipse or Sublime Text users, the [[https://github.com/GoClipse/goclipse][GoClipse]] and [[https://github.com/DisposaBoy/GoSublime][GoSublime]] projects add a gofmt facility to those editors. - -And for Git aficionados, the [[http://tip.golang.org/misc/git/pre-commit][misc/git/pre-commit script]] is a pre-commit hook that prevents incorrectly-formatted Go code from being committed. If you use Mercurial, the [[https://bitbucket.org/fhs/hgstyle/overview][hgstyle plugin]] provides a gofmt pre-commit hook. - -* Mechanical source transformation - -One of the greatest virtues of machine-formatted code is that it can be transformed mechanically without generating unrelated formatting noise in the diffs. Mechanical transformation is invaluable when working with large code bases, as it is both more comprehensive and less error prone than making wide-sweeping changes by hand. Indeed, when working at scale (like we do at Google) it often isn't practical to make these kinds of changes manually. - -The easiest way to mechanically manipulate Go code is with gofmt's -r flag. The flag specifies a rewrite rule of the form - - pattern -> replacement - -where both pattern and replacement are valid Go expressions. In the pattern, single-character lowercase identifiers serve as wildcards matching arbitrary sub-expressions, and those expressions are substituted for the same identifiers in the replacement. - -For example, this[[https://code.google.com/p/go/source/detail?r=ae4e014e0b77][ recent change]] to the Go core rewrote some uses of [[http://golang.org/pkg/bytes/#Compare][bytes.Compare]] to use the more efficient [[http://golang.org/pkg/bytes/#Equal][bytes.Equal]]. The contributor made the change using just two gofmt invocations: - - gofmt -r 'bytes.Compare(a, b) == 0 -> bytes.Equal(a, b)' - gofmt -r 'bytes.Compare(a, b) != 0 -> !bytes.Equal(a, b)' - -Gofmt also enables [[http://golang.org/cmd/fix/][gofix]], which can make arbitrarily complex source transformations. Gofix was an invaluable tool during the early days when we regularly made breaking changes to the language and libraries. For example, before Go 1 the built-in error interface didn't exist and the convention was to use the os.Error type. When we [[http://golang.org/doc/go1.html#errors][introduced error]], we provided a gofix module that rewrote all references to os.Error and its associated helper functions to use error and the new [[http://golang.org/pkg/errors/][errors package]]. It would have been daunting to attempt by hand, but with the code in a standard format it was relatively easy to prepare, execute, and review this change which touched almost all Go code in existence. - -For more about gofix, see [[http://blog.golang.org/2011/04/introducing-gofix.html][this article]]. diff --git a/content/go-for-app-engine-is-now-generally.article b/content/go-for-app-engine-is-now-generally.article deleted file mode 100644 index 21d1c59c..00000000 --- a/content/go-for-app-engine-is-now-generally.article +++ /dev/null @@ -1,19 +0,0 @@ -Go for App Engine is now generally available -21 Jul 2011 -Tags: appengine, release - -Andrew Gerrand - -* Introduction - -The Go and App Engine teams are excited to announce that the Go runtime for App Engine is now generally available. This means you can take that Go app you've been working on (or meaning to work on) and deploy it to App Engine right now with the new [[http://code.google.com/appengine/downloads.html][1.5.2 SDK]]. - -Since we announced the Go runtime at Google I/O we have continued to [[http://code.google.com/p/googleappengine/wiki/SdkForGoReleaseNotes][improve and extend]] Go support for the App Engine APIs and have added the Channels API. The Go Datastore API now supports transactions and ancestor queries, too. See the [[https://code.google.com/appengine/docs/go/][Go App Engine documentation]] for all the details. - -For those who have been using the Go SDK already, please note that the 1.5.2 release introduces `api_version` 2. This is because the new SDK is based on Go `release.r58.1` (the current stable version of Go) and is not backwards compatible with the previous release. Existing apps may require changes as per the [[http://golang.org/doc/devel/release.html#r58][r58 release notes]]. Once you've updated your code, you should redeploy your app with the line `api_version:`2` in its `app.yaml` file. Apps written against `api_version` 1 will stop working after the 18th of August. - -Finally, we owe a huge thanks to our trusted testers and their many bug reports. Their help was invaluable in reaching this important milestone. - -_The_fastest_way_to_get_started_with_Go_on_App_Engine_is_with_the_ [[http://code.google.com/appengine/docs/go/gettingstarted/][_Getting_Started_guide_]]. - -_Note_that_the_Go_runtime_is_still_considered_experimental;_it_is_not_as_well-supported_as_the_Python_and_Java_runtimes._ diff --git a/content/go-image-package.article b/content/go-image-package.article deleted file mode 100644 index 2f51dd4c..00000000 --- a/content/go-image-package.article +++ /dev/null @@ -1,194 +0,0 @@ -The Go image package -21 Sep 2011 -Tags: image, libraries, technical - -Nigel Tao - -* Introduction - -The [[http://golang.org/pkg/image/][image]] and [[http://golang.org/pkg/image/color/][image/color]] packages define a number of types: `color.Color` and `color.Model` describe colors, `image.Point` and `image.Rectangle` describe basic 2-D geometry, and `image.Image` brings the two concepts together to represent a rectangular grid of colors. A [[http://golang.org/doc/articles/image_draw.html][separate article]] covers image composition with the [[http://golang.org/pkg/image/draw/][image/draw]] package. - -* Colors and Color Models - -[[http://golang.org/pkg/image/color/#Color][Color]] is an interface that defines the minimal method set of any type that can be considered a color: one that can be converted to red, green, blue and alpha values. The conversion may be lossy, such as converting from CMYK or YCbCr color spaces. - - type Color interface { - // RGBA returns the alpha-premultiplied red, green, blue and alpha values - // for the color. Each value ranges within [0, 0xFFFF], but is represented - // by a uint32 so that multiplying by a blend factor up to 0xFFFF will not - // overflow. - RGBA() (r, g, b, a uint32) - } - -There are three important subtleties about the return values. First, the red, green and blue are alpha-premultiplied: a fully saturated red that is also 25% transparent is represented by RGBA returning a 75% r. Second, the channels have a 16-bit effective range: 100% red is represented by RGBA returning an r of 65535, not 255, so that converting from CMYK or YCbCr is not as lossy. Third, the type returned is `uint32`, even though the maximum value is 65535, to guarantee that multiplying two values together won't overflow. Such multiplications occur when blending two colors according to an alpha mask from a third color, in the style of [[https://en.wikipedia.org/wiki/Alpha_compositing][Porter and Duff's]] classic algebra: - - dstr, dstg, dstb, dsta := dst.RGBA() - srcr, srcg, srcb, srca := src.RGBA() - _, _, _, m := mask.RGBA() - const M = 1<<16 - 1 - // The resultant red value is a blend of dstr and srcr, and ranges in [0, M]. - // The calculation for green, blue and alpha is similar. - dstr = (dstr*(M-m) + srcr*m) / M - -The last line of that code snippet would have been more complicated if we worked with non-alpha-premultiplied colors, which is why `Color` uses alpha-premultiplied values. - -The image/color package also defines a number of concrete types that implement the `Color` interface. For example, [[http://golang.org/pkg/image/color/#RGBA][`RGBA`]] is a struct that represents the classic "8 bits per channel" color. - - type RGBA struct { - R, G, B, A uint8 - } - -Note that the `R` field of an `RGBA` is an 8-bit alpha-premultiplied color in the range [0, 255]. `RGBA` satisfies the `Color` interface by multiplying that value by 0x101 to generate a 16-bit alpha-premultiplied color in the range [0, 65535]. Similarly, the [[http://golang.org/pkg/image/color/#NRGBA][`NRGBA`]] struct type represents an 8-bit non-alpha-premultiplied color, as used by the PNG image format. When manipulating an `NRGBA`'s fields directly, the values are non-alpha-premultiplied, but when calling the `RGBA` method, the return values are alpha-premultiplied. - -A [[http://golang.org/pkg/image/color/#Model][`Model`]] is simply something that can convert `Color`s to other `Color`s, possibly lossily. For example, the `GrayModel` can convert any `Color` to a desaturated [[http://golang.org/pkg/image/color/#Gray][`Gray`]]. A `Palette` can convert any `Color` to one from a limited palette. - - type Model interface { - Convert(c Color) Color - } - - type Palette []Color - -* Points and Rectangles - -A [[http://golang.org/pkg/image/#Point][`Point`]] is an (x, y) co-ordinate on the integer grid, with axes increasing right and down. It is neither a pixel nor a grid square. A `Point` has no intrinsic width, height or color, but the visualizations below use a small colored square. - - type Point struct { - X, Y int - } - -.image go-image-package_image-package-01.png - - p := image.Point{2, 1} - -A [[http://golang.org/pkg/image/#Rectangle][`Rectangle`]] is an axis-aligned rectangle on the integer grid, defined by its top-left and bottom-right `Point`. A `Rectangle` also has no intrinsic color, but the visualizations below outline rectangles with a thin colored line, and call out their `Min` and `Max` `Point`s. - - type Rectangle struct { - Min, Max Point - } - -For convenience, `image.Rect(x0,`y0,`x1,`y1)` is equivalent to `image.Rectangle{image.Point{x0,`y0},`image.Point{x1,`y1}}`, but is much easier to type. - -A `Rectangle` is inclusive at the top-left and exclusive at the bottom-right. For a `Point`p` and a `Rectangle`r`, `p.In(r)` if and only if `r.Min.X`<=`p.X`&&`p.X`<`r.Max.X`, and similarly for `Y`. This is analagous to how a slice `s[i0:i1]` is inclusive at the low end and exclusive at the high end. (Unlike arrays and slices, a `Rectangle` often has a non-zero origin.) - -.image go-image-package_image-package-02.png - - r := image.Rect(2, 1, 5, 5) - // Dx and Dy return a rectangle's width and height. - fmt.Println(r.Dx(), r.Dy(), image.Pt(0, 0).In(r)) // prints 3 4 false - -Adding a `Point` to a `Rectangle` translates the `Rectangle`. Points and Rectangles are not restricted to be in the bottom-right quadrant. - -.image go-image-package_image-package-03.png - - r := image.Rect(2, 1, 5, 5).Add(image.Pt(-4, -2)) - fmt.Println(r.Dx(), r.Dy(), image.Pt(0, 0).In(r)) // prints 3 4 true - -Intersecting two Rectangles yields another Rectangle, which may be empty. - -.image go-image-package_image-package-04.png - - r := image.Rect(0, 0, 4, 3).Intersect(image.Rect(2, 2, 5, 5)) - // Size returns a rectangle's width and height, as a Point. - fmt.Printf("%#v\n", r.Size()) // prints image.Point{X:2, Y:1} - -Points and Rectangles are passed and returned by value. A function that takes a `Rectangle` argument will be as efficient as a function that takes two `Point` arguments, or four `int` arguments. - -* Images - -An [[http://golang.org/pkg/image/#Image][Image]] maps every grid square in a `Rectangle` to a `Color` from a `Model`. "The pixel at (x, y)" refers to the color of the grid square defined by the points (x, y), (x+1, y), (x+1, y+1) and (x, y+1). - - type Image interface { - // ColorModel returns the Image's color model. - ColorModel() color.Model - // Bounds returns the domain for which At can return non-zero color. - // The bounds do not necessarily contain the point (0, 0). - Bounds() Rectangle - // At returns the color of the pixel at (x, y). - // At(Bounds().Min.X, Bounds().Min.Y) returns the upper-left pixel of the grid. - // At(Bounds().Max.X-1, Bounds().Max.Y-1) returns the lower-right one. - At(x, y int) color.Color - } - -A common mistake is assuming that an `Image`'s bounds start at (0, 0). For example, an animated GIF contains a sequence of Images, and each `Image` after the first typically only holds pixel data for the area that changed, and that area doesn't necessarily start at (0, 0). The correct way to iterate over an `Image` m's pixels looks like: - - b := m.Bounds() - for y := b.Min.Y; y < b.Max.Y; y++ { - for x := b.Min.X; x < b.Max.X; x++ { - doStuffWith(m.At(x, y)) - } - } - -`Image` implementations do not have to be based on an in-memory slice of pixel data. For example, a [[http://golang.org/pkg/image/#Uniform][`Uniform`]] is an `Image` of enormous bounds and uniform color, whose in-memory representation is simply that color. - - type Uniform struct { - C color.Color - } - -Typically, though, programs will want an image based on a slice. Struct types like [[http://golang.org/pkg/image/#RGBA][`RGBA`]] and [[http://golang.org/pkg/image/#Gray][`Gray`]] (which other packages refer to as `image.RGBA` and `image.Gray`) hold slices of pixel data and implement the `Image` interface. - - type RGBA struct { - // Pix holds the image's pixels, in R, G, B, A order. The pixel at - // (x, y) starts at Pix[(y-Rect.Min.Y)*Stride + (x-Rect.Min.X)*4]. - Pix []uint8 - // Stride is the Pix stride (in bytes) between vertically adjacent pixels. - Stride int - // Rect is the image's bounds. - Rect Rectangle - } - -These types also provide a `Set(x,`y`int,`c`color.Color)` method that allows modifying the image one pixel at a time. - - m := image.NewRGBA(image.Rect(0, 0, 640, 480)) - m.Set(5, 5, color.RGBA{255, 0, 0, 255}) - -If you're reading or writing a lot of pixel data, it can be more efficient, but more complicated, to access these struct type's `Pix` field directly. - -The slice-based `Image` implementations also provide a `SubImage` method, which returns an `Image` backed by the same array. Modifying the pixels of a sub-image will affect the pixels of the original image, analagous to how modifying the contents of a sub-slice `s[i0:i1]` will affect the contents of the original slice `s`. - -.image go-image-package_image-package-05.png - - m0 := image.NewRGBA(image.Rect(0, 0, 8, 5)) - m1 := m0.SubImage(image.Rect(1, 2, 5, 5)).(*image.RGBA) - fmt.Println(m0.Bounds().Dx(), m1.Bounds().Dx()) // prints 8, 4 - fmt.Println(m0.Stride == m1.Stride) // prints true - -For low-level code that works on an image's `Pix` field, be aware that ranging over `Pix` can affect pixels outside an image's bounds. In the example above, the pixels covered by `m1.Pix` are shaded in blue. Higher-level code, such as the `At` and `Set` methods or the [[http://golang.org/pkg/image/draw/][image/draw package]], will clip their operations to the image's bounds. - -* Image Formats - -The standard package library supports a number of common image formats, such as GIF, JPEG and PNG. If you know the format of a source image file, you can decode from an [[http://golang.org/pkg/io/#Reader][`io.Reader`]] directly. - - import ( - "image/jpeg" - "image/png" - "io" - ) - - // convertJPEGToPNG converts from JPEG to PNG. - func convertJPEGToPNG(w io.Writer, r io.Reader) error { - img, err := jpeg.Decode(r) - if err != nil { - return err - } - return png.Encode(w, img) - } - -If you have image data of unknown format, the [[http://golang.org/pkg/image/#Decode][`image.Decode`]] function can detect the format. The set of recognized formats is constructed at run time and is not limited to those in the standard package library. An image format package typically registers its format in an init function, and the main package will "underscore import" such a package solely for the side effect of format registration. - - import ( - "image" - "image/png" - "io" - - _ "code.google.com/p/vp8-go/webp" - _ "image/jpeg" - ) - - // convertToPNG converts from any recognized format to PNG. - func convertToPNG(w io.Writer, r io.Reader) error { - img, _, err := image.Decode(r) - if err != nil { - return err - } - return png.Encode(w, img) - } diff --git a/content/go-image-package_image-package-01.png b/content/go-image-package_image-package-01.png deleted file mode 100644 index aad9b124..00000000 Binary files a/content/go-image-package_image-package-01.png and /dev/null differ diff --git a/content/go-image-package_image-package-02.png b/content/go-image-package_image-package-02.png deleted file mode 100644 index 3dd4692f..00000000 Binary files a/content/go-image-package_image-package-02.png and /dev/null differ diff --git a/content/go-image-package_image-package-03.png b/content/go-image-package_image-package-03.png deleted file mode 100644 index 5bc0bf73..00000000 Binary files a/content/go-image-package_image-package-03.png and /dev/null differ diff --git a/content/go-image-package_image-package-04.png b/content/go-image-package_image-package-04.png deleted file mode 100644 index 393dc120..00000000 Binary files a/content/go-image-package_image-package-04.png and /dev/null differ diff --git a/content/go-image-package_image-package-05.png b/content/go-image-package_image-package-05.png deleted file mode 100644 index 54c47b67..00000000 Binary files a/content/go-image-package_image-package-05.png and /dev/null differ diff --git a/content/go-imagedraw-package.article b/content/go-imagedraw-package.article deleted file mode 100644 index 0c985602..00000000 --- a/content/go-imagedraw-package.article +++ /dev/null @@ -1,147 +0,0 @@ -The Go image/draw package -29 Sep 2011 -Tags: draw, image, libraries, technical - -Nigel Tao - -* Introduction - -[[http://golang.org/pkg/image/draw/][Package image/draw]] defines only one operation: drawing a source image onto a destination image, through an optional mask image. This one operation is surprisingly versatile and can perform a number of common image manipulation tasks elegantly and efficiently. - -Composition is performed pixel by pixel in the style of the Plan 9 graphics library and the X Render extension. The model is based on the classic "Compositing Digital Images" paper by Porter and Duff, with an additional mask parameter: `dst`=`(src`IN`mask)`OP`dst`. For a fully opaque mask, this reduces to the original Porter-Duff formula: `dst`=`src`OP`dst`. In Go, a nil mask image is equivalent to an infinitely sized, fully opaque mask image. - -The Porter-Duff paper presented [[http://www.w3.org/TR/SVGCompositing/examples/compop-porterduff-examples.png][12 different composition operators]], but with an explicit mask, only 2 of these are needed in practice: source-over-destination and source. In Go, these operators are represented by the `Over` and `Src` constants. The `Over` operator performs the natural layering of a source image over a destination image: the change to the destination image is smaller where the source (after masking) is more transparent (that is, has lower alpha). The `Src` operator merely copies the source (after masking) with no regard for the destination image's original content. For fully opaque source and mask images, the two operators produce the same output, but the `Src` operator is usually faster. - -* Geometric Alignment - -Composition requires associating destination pixels with source and mask pixels. Obviously, this requires destination, source and mask images, and a composition operator, but it also requires specifying what rectangle of each image to use. Not every drawing should write to the entire destination: when updating an animating image, it is more efficient to only draw the parts of the image that have changed. Not every drawing should read from the entire source: when using a sprite that combines many small images into one large one, only a part of the image is needed. Not every drawing should read from the entire mask: a mask image that collects a font's glyphs is similar to a sprite. Thus, drawing also needs to know three rectangles, one for each image. Since each rectangle has the same width and height, it suffices to pass a destination rectangle `r` and two points `sp` and `mp`: the source rectangle is equal to `r` translated so that `r.Min` in the destination image aligns with `sp` in the source image, and similarly for `mp`. The effective rectangle is also clipped to each image's bounds in their respective co-ordinate space. - -.image go-imagedraw-package_20.png - -The [[http://golang.org/pkg/image/draw/#DrawMask][`DrawMask`]] function takes seven arguments, but an explicit mask and mask-point are usually unnecessary, so the [[http://golang.org/pkg/image/draw/#Draw][`Draw`]] function takes five: - - // Draw calls DrawMask with a nil mask. - func Draw(dst Image, r image.Rectangle, src image.Image, sp image.Point, op Op) - func DrawMask(dst Image, r image.Rectangle, src image.Image, sp image.Point, - mask image.Image, mp image.Point, op Op) - -The destination image must be mutable, so the image/draw package defines a [[http://golang.org/pkg/image/draw/#Image][`draw.Image`]] interface which has a `Set` method. - - type Image interface { - image.Image - Set(x, y int, c color.Color) - } - - -* Filling a Rectangle - -To fill a rectangle with a solid color, use an `image.Uniform` source. The `ColorImage` type re-interprets a `Color` as a practically infinite-sized `Image` of that color. For those familiar with the design of Plan 9's draw library, there is no need for an explicit "repeat bit" in Go's slice-based image types; the concept is subsumed by `Uniform`. - - // image.ZP is the zero point -- the origin. - draw.Draw(dst, r, &image.Uniform{c}, image.ZP, draw.Src) - -To initialize a new image to all-blue: - - m := image.NewRGBA(image.Rect(0, 0, 640, 480)) - blue := color.RGBA{0, 0, 255, 255} - draw.Draw(m, m.Bounds(), &image.Uniform{blue}, image.ZP, draw.Src) - -To reset an image to transparent (or black, if the destination image's color model cannot represent transparency), use `image.Transparent`, which is an `image.Uniform`: - - draw.Draw(m, m.Bounds(), image.Transparent, image.ZP, draw.Src) - - -.image go-imagedraw-package_2a.png - - -* Copying an Image - -To copy from a rectangle `sr` in the source image to a rectangle starting at a point `dp` in the destination, convert the source rectangle into the destination image's co-ordinate space: - - r := image.Rectangle{dp, dp.Add(sr.Size())} - draw.Draw(dst, r, src, sr.Min, draw.Src) - - -Alternatively: - - r := sr.Sub(sr.Min).Add(dp) - draw.Draw(dst, r, src, sr.Min, draw.Src) - - -To copy the entire source image, use `sr`=`src.Bounds()`. - - -.image go-imagedraw-package_2b.png - - -* Scrolling an Image - -Scrolling an image is just copying an image to itself, with different destination and source rectangles. Overlapping destination and source images are perfectly valid, just as Go's built-in copy function can handle overlapping destination and source slices. To scroll an image m by 20 pixels: - - b := m.Bounds() - p := image.Pt(0, 20) - // Note that even though the second argument is b, - // the effective rectangle is smaller due to clipping. - draw.Draw(m, b, m, b.Min.Add(p), draw.Src) - dirtyRect := b.Intersect(image.Rect(b.Min.X, b.Max.Y-20, b.Max.X, b.Max.Y)) - - -.image go-imagedraw-package_2c.png - - -* Converting an Image to RGBA - -The result of decoding an image format might not be an `image.RGBA`: decoding a GIF results in an `image.Paletted`, decoding a JPEG results in a `ycbcr.YCbCr`, and the result of decoding a PNG depends on the image data. To convert any image to an `image.RGBA`: - - b := src.Bounds() - m := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) - draw.Draw(m, m.Bounds(), src, b.Min, draw.Src) - - -.image go-imagedraw-package_2d.png - -* Drawing Through a Mask - -To draw an image through a circular mask with center `p` and radius `r`: - - type circle struct { - p image.Point - r int - } - - func (c *circle) ColorModel() color.Model { - return color.AlphaModel - } - - func (c *circle) Bounds() image.Rectangle { - return image.Rect(c.p.X-c.r, c.p.Y-c.r, c.p.X+c.r, c.p.Y+c.r) - } - - func (c *circle) At(x, y int) color.Color { - xx, yy, rr := float64(x-c.p.X)+0.5, float64(y-c.p.Y)+0.5, float64(c.r) - if xx*xx+yy*yy < rr*rr { - return color.Alpha{255} - } - return color.Alpha{0} - } - - draw.DrawMask(dst, dst.Bounds(), src, image.ZP, &circle{p, r}, image.ZP, draw.Over) - - -.image go-imagedraw-package_2e.png - -* Drawing Font Glyphs - -To draw a font glyph in blue starting from a point `p`, draw with an `image.ColorImage` source and an `image.Alpha`mask`. For simplicity, we aren't performing any sub-pixel positioning or rendering, or correcting for a font's height above a baseline. - - src := &image.Uniform{color.RGBA{0, 0, 255, 255}} - mask := theGlyphImageForAFont() - mr := theBoundsFor(glyphIndex) - draw.DrawMask(dst, mr.Sub(mr.Min).Add(p), src, image.ZP, mask, mr.Min, draw.Over) - -.image go-imagedraw-package_2f.png - - -* Performance - -The image/draw package implementation demonstrates how to provide an image manipulation function that is both general purpose, yet efficient for common cases. The `DrawMask` function takes arguments of interface types, but immediately makes type assertions that its arguments are of specific struct types, corresponding to common operations like drawing one `image.RGBA` image onto another, or drawing an `image.Alpha` mask (such as a font glyph) onto an `image.RGBA` image. If a type assertion succeeds, that type information is used to run a specialized implementation of the general algorithm. If the assertions fail, the fallback code path uses the generic `At` and `Set` methods. The fast-paths are purely a performance optimization; the resultant destination image is the same either way. In practice, only a small number of special cases are necessary to support typical applications. diff --git a/content/go-imagedraw-package_20.png b/content/go-imagedraw-package_20.png deleted file mode 100644 index 063e4306..00000000 Binary files a/content/go-imagedraw-package_20.png and /dev/null differ diff --git a/content/go-imagedraw-package_2a.png b/content/go-imagedraw-package_2a.png deleted file mode 100644 index 3f1c0aff..00000000 Binary files a/content/go-imagedraw-package_2a.png and /dev/null differ diff --git a/content/go-imagedraw-package_2b.png b/content/go-imagedraw-package_2b.png deleted file mode 100644 index 32b24701..00000000 Binary files a/content/go-imagedraw-package_2b.png and /dev/null differ diff --git a/content/go-imagedraw-package_2c.png b/content/go-imagedraw-package_2c.png deleted file mode 100644 index f9abce5b..00000000 Binary files a/content/go-imagedraw-package_2c.png and /dev/null differ diff --git a/content/go-imagedraw-package_2d.png b/content/go-imagedraw-package_2d.png deleted file mode 100644 index ed0a9f92..00000000 Binary files a/content/go-imagedraw-package_2d.png and /dev/null differ diff --git a/content/go-imagedraw-package_2e.png b/content/go-imagedraw-package_2e.png deleted file mode 100644 index 483b208e..00000000 Binary files a/content/go-imagedraw-package_2e.png and /dev/null differ diff --git a/content/go-imagedraw-package_2f.png b/content/go-imagedraw-package_2f.png deleted file mode 100644 index 3dce02d5..00000000 Binary files a/content/go-imagedraw-package_2f.png and /dev/null differ diff --git a/content/go-maps-in-action.article b/content/go-maps-in-action.article deleted file mode 100644 index 80c1477b..00000000 --- a/content/go-maps-in-action.article +++ /dev/null @@ -1,185 +0,0 @@ -Go maps in action -6 Feb 2013 -Tags: map, technical - -Andrew Gerrand - -* Introduction - -One of the most useful data structures in computer science is the hash table. Many hash table implementations exist with varying properties, but in general they offer fast lookups, adds, and deletes. Go provides a built-in map type that implements a hash table. - -* Declaration and initialization - -A Go map type looks like this: - - map[KeyType]ValueType - -where `KeyType` may be any type that is [[http://golang.org/ref/spec#Comparison_operators][comparable]] (more on this later), and `ValueType` may be any type at all, including another map! - -This variable `m` is a map of string keys to int values: - - var m map[string]int - -Map types are reference types, like pointers or slices, and so the value of `m` above is `nil`; it doesn't point to an initialized map. A nil map behaves like an empty map when reading, but attempts to write to a nil map will cause a runtime panic; don't do that. To initialize a map, use the built in `make` function: - - m = make(map[string]int) - -The `make` function allocates and initializes a hash map data structure and returns a map value that points to it. The specifics of that data structure are an implementation detail of the runtime and are not specified by the language itself. In this article we will focus on the _use_ of maps, not their implementation. - -* Working with maps - -Go provides a familiar syntax for working with maps. This statement sets the key `"route"` to the value `66`: - - m["route"] = 66 - -This statement retrieves the value stored under the key `"route"` and assigns it to a new variable i: - - i := m["route"] - -If the requested key doesn't exist, we get the value type's _zero_value_. In this case the value type is `int`, so the zero value is `0`: - - j := m["root"] - // j == 0 - -The built in `len` function returns on the number of items in a map: - - n := len(m) - -The built in `delete` function removes an entry from the map: - - delete(m, "route") - -The `delete` function doesn't return anything, and will do nothing if the specified key doesn't exist. - -A two-value assignment tests for the existence of a key: - - i, ok := m["route"] - -In this statement, the first value (`i`) is assigned the value stored under the key `"route"`. If that key doesn't exist, `i` is the value type's zero value (`0`). The second value (`ok`) is a `bool` that is `true` if the key exists in the map, and `false` if not. - -To test for a key without retrieving the value, use an underscore in place of the first value: - - _, ok := m["route"] - -To iterate over the contents of a map, use the `range` keyword: - - for key, value := range m { - fmt.Println("Key:", key, "Value:", value) - } - -To initialize a map with some data, use a map literal: - - commits := map[string]int{ - "rsc": 3711, - "r": 2138, - "gri": 1908, - "adg": 912, - } - -The same syntax may be used to initialize an empty map, which is functionally identical to using the `make` function: - - m = map[string]int{} - -* Exploiting zero values - -It can be convenient that a map retrieval yields a zero value when the key is not present. - -For instance, a map of boolean values can be used as a set-like data structure (recall that the zero value for the boolean type is false). This example traverses a linked list of `Nodes` and prints their values. It uses a map of `Node` pointers to detect cycles in the list. - -.code go-maps-in-action/list.go /START/,/END/ - -The expression `visited[n]` is `true` if `n` has been visited, or `false` if `n` is not present. There's no need to use the two-value form to test for the presence of `n` in the map; the zero value default does it for us. - -Another instance of helpful zero values is a map of slices. Appending to a nil slice just allocates a new slice, so it's a one-liner to append a value to a map of slices; there's no need to check if the key exists. In the following example, the slice people is populated with `Person` values. Each `Person` has a `Name` and a slice of Likes. The example creates a map to associate each like with a slice of people that like it. - -.code go-maps-in-action/people.go /START1/,/END1/ - -To print a list of people who like cheese: - -.code go-maps-in-action/people.go /START2/,/END2/ - -To print the number of people who like bacon: - -.code go-maps-in-action/people.go /bacon/ - -Note that since both range and len treat a nil slice as a zero-length slice, these last two examples will work even if nobody likes cheese or bacon (however unlikely that may be). - -* Key types - -As mentioned earlier, map keys may be of any type that is comparable. The [[http://golang.org/ref/spec#Comparison_operators][language spec]] defines this precisely, but in short, comparable types are boolean, numeric, string, pointer, channel, and interface types, and structs or arrays that contain only those types. Notably absent from the list are slices, maps, and functions; these types cannot be compared using `==`, and may not be used as map keys. - -It's obvious that strings, ints, and other basic types should be available as map keys, but perhaps unexpected are struct keys. Struct can be used to key data by multiple dimensions. For example, this map of maps could be used to tally web page hits by country: - - hits := make(map[string]map[string]int) - -This is map of string to (map of `string` to `int`). Each key of the outer map is the path to a web page with its own inner map. Each inner map key is a two-letter country code. This expression retrieves the number of times an Australian has loaded the documentation page: - - n := hits["/doc/"]["au"] - -Unfortunately, this approach becomes unwieldy when adding data, as for any given outer key you must check if the inner map exists, and create it if needed: - - func add(m map[string]map[string]int, path, country string) { - mm, ok := m[path] - if !ok { - mm = make(map[string]int) - m[path] = mm - } - mm[country]++ - } - add(hits, "/doc/", "au") - -On the other hand, a design that uses a single map with a struct key does away with all that complexity: - - type Key struct { - Path, Country string - } - hits := make(map[Key]int) - -When an Vietnamese person visits the home page, incrementing (and possibly creating) the appropriate counter is a one-liner: - - hits[Key{"/", "vn"}]++ - -And it's similarly straightforward to see how many Swiss people have read the spec: - - n := hits[Key{"/ref/spec", "ch"}] - -* Concurrency - -[[http://golang.org/doc/faq#atomic_maps][Maps are not safe for concurrent use]]: it's not defined what happens when you read and write to them simultaneously. If you need to read from and write to a map from concurrently executing goroutines, the accesses must be mediated by some kind of synchronization mechanism. One common way to protect maps is with [[http://golang.org/pkg/sync/#RWMutex][sync.RWMutex]]. - -This statement declares a `counter` variable that is an anonymous struct containing a map and an embedded `sync.RWMutex`. - - var counter = struct{ - sync.RWMutex - m map[string]int - }{m: make(map[string]int)} - -To read from the counter, take the read lock: - - - counter.RLock() - n := counter.m["some_key"] - counter.RUnlock() - fmt.Println("some_key:", n) - -To write to the counter, take the write lock: - - counter.Lock() - counter.m["some_key"]++ - counter.Unlock() - -* Iteration order - -When iterating over a map with a range loop, the iteration order is not specified and is not guaranteed to be the same from one iteration to the next. Since Go 1 the runtime randomizes map iteration order, as programmers relied on the stable iteration order of the previous implementation. If you require a stable iteration order you must maintain a separate data structure that specifies that order. This example uses a separate sorted slice of keys to print a `map[int]string` in key order: - - import "sort" - - var m map[int]string - var keys []int - for k := range m { - keys = append(keys, k) - } - sort.Ints(keys) - for _, k := range keys { - fmt.Println("Key:", k, "Value:", m[k]) - } diff --git a/content/go-maps-in-action/list.go b/content/go-maps-in-action/list.go deleted file mode 100644 index 02bea2c5..00000000 --- a/content/go-maps-in-action/list.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build OMIT - -package main - -import "fmt" - -func main() { - // START OMIT - type Node struct { - Next *Node - Value interface{} - } - var first *Node - - visited := make(map[*Node]bool) // HL - for n := first; n != nil; n = n.Next { - if visited[n] { // HL - fmt.Println("cycle detected") - break - } - visited[n] = true // HL - fmt.Println(n.Value) - } - // END OMIT -} diff --git a/content/go-maps-in-action/people.go b/content/go-maps-in-action/people.go deleted file mode 100644 index ff5d9e3a..00000000 --- a/content/go-maps-in-action/people.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build OMIT - -package main - -import "fmt" - -func main() { - // START1 OMIT - type Person struct { - Name string - Likes []string - } - var people []*Person - - likes := make(map[string][]*Person) // HL - for _, p := range people { - for _, l := range p.Likes { - likes[l] = append(likes[l], p) // HL - } - } - // END1 OMIT - - // START2 OMIT - for _, p := range likes["cheese"] { - fmt.Println(p.Name, "likes cheese.") - } - // END2 OMIT - - fmt.Println(len(likes["bacon"]), "people like bacon.") -} diff --git a/content/go-one-year-ago-today.article b/content/go-one-year-ago-today.article deleted file mode 100644 index f471c54b..00000000 --- a/content/go-one-year-ago-today.article +++ /dev/null @@ -1,41 +0,0 @@ -Go: one year ago today -10 Nov 2010 -Tags: birthday - -Andrew Gerrand - -* Introduction - -On the 10th of November 2009 we launched the Go project: an open-source programming language with a focus on simplicity and efficiency. The intervening year has seen a great many developments both in the Go project itself and in its community. - -We set out to build a language for systems programming - the kinds of programs one might typically write in C or C++ - and we were surprised by Go’s utility as a general purpose language. We had anticipated interest from C, C++, and Java programmers, but the flurry of interest from users of dynamically-typed languages like Python and JavaScript was unexpected. Go’s combination of native compilation, static typing, memory management, and lightweight syntax seemed to strike a chord with a broad cross-section of the programming community. - -That cross-section grew to become a dedicated community of enthusiastic Go coders. Our [[http://groups.google.com/group/golang-nuts][mailing list]] has over 3,800 members, with around 1,500 posts each month. The project has over 130 [[http://golang.org/CONTRIBUTORS][contributors]] (people who have submitted code or documentation), and of the 2,800 commits since launch almost one third were contributed by programmers outside the core team. To get all that code into shape, nearly 14,000 emails were exchanged on our [[http://groups.google.com/group/golang-dev][development mailing list]]. - -Those numbers reflect a labor whose fruits are evident in the project’s code base. The compilers have improved substantially, with faster and more efficient code generation, more than one hundred reported bugs fixed, and support for a widening range of operating systems and architectures. The Windows port is approaching completion thanks to a dedicated group of contributors (one of whom became our first non-Google committer to the project). The ARM port has also made great progress, recently reaching the milestone of passing all tests. - -The Go tool set has been expanded and improved. The Go documentation tool, [[http://golang.org/cmd/godoc/][godoc]], now supports the documentation of other source trees (you can browse and search your own code) and provides a [[http://golang.org/doc/codewalk/]["code walk"]] interface for presenting tutorial materials (among many more improvements). [[http://golang.org/cmd/goinstall/][Goinstall]] , a new package management tool, allows users to install and update external packages with a single command. [[http://golang.org/cmd/gofmt/][Gofmt]], the Go pretty-printer, now makes syntactic simplifications where possible. [[http://golang.org/misc/goplay/][Goplay]], a web-based “compile-as-you-type” tool, is a convenient way to experiment with Go for those times when you don’t have access to the [[http://golang.org/doc/play/][Go Playground]]. - -The standard library has grown by over 42,000 lines of code and includes 20 new [[http://golang.org/pkg/][packages]]. Among the additions are the [[http://golang.org/pkg/image/jpeg/][jpeg]], [[http://golang.org/pkg/rpc/jsonrpc/][jsonrpc]], [[http://golang.org/pkg/mime/][mime]], [[http://golang.org/pkg/netchan/][netchan]], and [[http://golang.org/pkg/smtp/][smtp]] packages, as well as a slew of new [[http://golang.org/pkg/crypto/][cryptography]] packages. More generally, the standard library has been continuously refined and revised as our understanding of Go’s idioms deepens. - -The debugging story has gotten better, too. Recent improvements to the DWARF output of the gc compilers make the GNU debugger, GDB, useful for Go binaries, and we’re actively working on making that debugging information more complete. (See the [[http://blog.golang.org/2010/11/debugging-go-code-status-report.html][ recent blog post]] for details.) - -It’s now easier than ever to link against existing libraries written in languages other than Go. Go support is in the most recent [[http://www.swig.org/][SWIG]] release, version 2.0.1, making it easier to link against C and C++ code, and our [[http://golang.org/cmd/cgo/][cgo]] tool has seen many fixes and improvements. - -[[http://golang.org/doc/gccgo_install.html][Gccgo]], the Go front end for the GNU C Compiler, has kept pace with the gc compiler as a parallel Go implementation. It now has a working garbage collector, and has been accepted into the GCC core. We’re now working toward making [[http://code.google.com/p/gofrontend/][gofrontend]] available as a BSD-licensed Go compiler front end, fully decoupled from GCC. - -Outside the Go project itself Go is starting to be used to build real software. There are more than 200 Go programs and libraries listed on our [[http://godashboard.appspot.com/project][Project dashboard]], and hundreds more on [[http://code.google.com/hosting/search?q=label:Go][Google Code]] and [[https://github.com/search?q=language:Go][Github]]. On our mailing list and IRC channel you can find coders from around the world who use Go for their programming projects. (See our [[http://blog.golang.org/2010/10/real-go-projects-smarttwitter-and-webgo.html][guest blog post]] from last month for a real-world example.) Internally at Google there are several teams that choose Go for building production software, and we have received reports from other companies that are developing sizable systems in Go. We have also been in touch with several educators who are using Go as a teaching language. - -The language itself has grown and matured, too. In the past year we have received many feature requests. But Go is a small language, and we’ve worked hard to ensure that any new feature strikes the right compromise between simplicity and utility. Since the launch we have made a number of language changes, many of which were driven by feedback from the community. - -- Semicolons are now optional in almost all instances. [[http://golang.org/doc/go_spec.html#Semicolons][spec]] -- The new built-in functions `copy` and `append` make management of slices more efficient and straightforward. [[http://golang.org/doc/go_spec.html#Appending_and_copying_slices][spec]] -- The upper and lower bounds may be omitted when making a sub-slice. This means that `s[:]` is shorthand for `s[0:len(s)]`. [[http://golang.org/doc/go_spec.html#Slices][spec]] -- The new built-in function `recover` complements `panic` and `defer` as an error handling mechanism. [[http://blog.golang.org/2010/08/defer-panic-and-recover.html][blog]], [[http://golang.org/doc/go_spec.html#Handling_panics][spec]] -- The new complex number types (`complex`, `complex64`, and `complex128`) simplify certain mathematical operations. [[http://golang.org/doc/go_spec.html#Complex_numbers][spec]], [[http://golang.org/doc/go_spec.html#Imaginary_literals][spec]] -- The composite literal syntax permits the omission of redundant type information (when specifying two-dimensional arrays, for example). [[http://golang.org/doc/devel/release.html#2010-10-27][release.2010-10-27]], [[http://golang.org/doc/go_spec.html#Composite_literals][spec]] -- A general syntax for variable function arguments (`...T`) and their propagation (`v...`) is now specified. [[http://golang.org/doc/go_spec.html#Function_Types][spec]], [[http://golang.org/doc/go_spec.html#Passing_arguments_to_..._parameters][ spec]], [[http://golang.org/doc/devel/release.html#2010-09-29][release.2010-09-29]] - -Go is certainly ready for production use, but there is still room for improvement. Our focus for the immediate future is making Go programs faster and more efficient in the context of high performance systems. This means improving the garbage collector, optimizing generated code, and improving the core libraries. We’re also exploring some further additions to the type system to make generic programming easier. A lot has happened in a year; it’s been both thrilling and satisfying. We hope that this coming year will be even more fruitful than the last. - -_If_you’ve_been_meaning_to_get_[back]_into_Go,_now_is_a_great_time_to_do_so!_Check_out_the_ [[http://golang.org/doc/docs.html][_Documentation_]] _and_ [[http://golang.org/doc/install.html][_Getting_Started_]] _pages_for_more_information,_or_just_go_nuts_in_the_ [[http://golang.org/doc/play/][_Go_Playground_]]. diff --git a/content/go-programming-language-turns-two.article b/content/go-programming-language-turns-two.article deleted file mode 100644 index c835ec1b..00000000 --- a/content/go-programming-language-turns-two.article +++ /dev/null @@ -1,28 +0,0 @@ -The Go Programming Language turns two -10 Nov 2011 -Tags: appengine, community, gopher - -Andrew Gerrand - -* Introduction - -Two years ago a small team at Google went public with their fledgling project - the Go Programming Language. They presented a language spec, two compilers, a modest standard library, some novel tools, and plenty of accurate (albeit succinct) documentation. They watched with excitement as programmers around the world began to play with Go. The team continued to iterate and improve on what they had built, and were gradually joined by dozens - and then hundreds - of programmers from the open source community. -The Go Authors went on to produce lots of libraries, new tools, and reams of [[http://golang.org/doc/docs.html][documentation]]. They celebrated a successful year in the public eye with a [[http://blog.golang.org/2010/11/go-one-year-ago-today.html][blog post]] last November that concluded "Go is certainly ready for production use, but there is still room for improvement. Our focus for the immediate future is making Go programs faster and more efficient in the context of high performance systems." - -Today is the second anniversary of Go's release, and Go is faster and more stable than ever. Careful tuning of Go's code generators, concurrency primitives, garbage collector, and core libraries have increased the performance of Go programs, and native support for [[http://blog.golang.org/2011/06/profiling-go-programs.html][profiling]] and [[http://blog.golang.org/2011/10/debugging-go-programs-with-gnu-debugger.html][debugging]] makes it easier to detect and remove performance issues in user code. Go is also now easier to learn with [[http://tour.golang.org/][A Tour of Go]], an interactive tutorial you can take from the comfort of your web browser. - -This year we introduced the experimental [[http://code.google.com/appengine/docs/go/][Go runtime]] for Google's App Engine platform, and we have been steadily increasing the Go runtime's support for App Engine's APIs. Just this week we released [[http://code.google.com/appengine/downloads.html][version 1.6.0]] of the Go App Engine SDK, which includes support for [[http://code.google.com/appengine/docs/go/backends/overview.html][backends]] (long-running processes), finer control over datastore indexes, and various other improvements. Today, the Go runtime is near feature parity with - and is a viable alternative to - the Python and Java runtimes. In fact, we now serve [[http://golang.org/][golang.org]] by running a version of [[http://golang.org/cmd/godoc/][godoc]] on the App Engine service. - -While 2010 was a year of discovery and experimentation, 2011 was a year of fine tuning and planning for the future. This year we issued several "[[http://golang.org/doc/devel/release.html][release]]" versions of Go that were more reliable and better supported than weekly snapshots. We also introduced [[http://golang.org/cmd/gofix/][gofix]] to take the pain out of migrating to newer releases. Furthermore, last month we announced a [[http://blog.golang.org/2011/10/preview-of-go-version-1.html][plan for Go version 1]] - a release that will be supported for years to come. Work toward Go 1 is already underway and you can observe our progress by the latest weekly snapshot at [[http://weekly.golang.org/pkg/][weekly.golang.org]]. - -The plan is to launch Go 1 in early 2012. We hope to bring the Go App Engine runtime out of "experimental" status at the same time. - -But that's not all. 2011 was an exciting year for the gopher, too. He has manifested himself as a plush toy (a highly prized gift at Google I/O and other Go talks) and in vinyl form (received by every attendee at OSCON and now available at the [[http://www.googlestore.com/Fun/Go+Gopher+Figurine.axd][Google Store]]). - -.image go-programming-language-turns-two_gophers.jpg - -And, most surprisingly, at Halloween he made an appearance with his gopher girlfriend! - -.image go-programming-language-turns-two_costume.jpg - -Photograph by [[https://plus.google.com/106640494112897458359/posts][Chris Nokleberg]]. diff --git a/content/go-programming-language-turns-two_costume.jpg b/content/go-programming-language-turns-two_costume.jpg deleted file mode 100644 index aa86a6c6..00000000 Binary files a/content/go-programming-language-turns-two_costume.jpg and /dev/null differ diff --git a/content/go-programming-language-turns-two_gophers.jpg b/content/go-programming-language-turns-two_gophers.jpg deleted file mode 100644 index 72075da3..00000000 Binary files a/content/go-programming-language-turns-two_gophers.jpg and /dev/null differ diff --git a/content/go-programming-session-video-from.article b/content/go-programming-session-video-from.article deleted file mode 100644 index 0d44f9e3..00000000 --- a/content/go-programming-session-video-from.article +++ /dev/null @@ -1,11 +0,0 @@ -Go Programming session video from Google I/O -6 Jun 2010 -Tags: video, talk - -Andrew Gerrand - -* Introduction - -Below is the video of the talk given by Rob Pike and Russ Cox at Google I/O 2010. - -.iframe http://www.youtube.com/embed/jgVhBThJdXc 304 540 diff --git a/content/go-slices-usage-and-internals.article b/content/go-slices-usage-and-internals.article deleted file mode 100644 index d4a50fde..00000000 --- a/content/go-slices-usage-and-internals.article +++ /dev/null @@ -1,233 +0,0 @@ -Go Slices: usage and internals -5 Jan 2011 -Tags: slice, technical - -Andrew Gerrand - -* Introduction - -Go's slice type provides a convenient and efficient means of working with sequences of typed data. Slices are analogous to arrays in other languages, but have some unusual properties. This article will look at what slices are and how they are used. - -* Arrays - -The slice type is an abstraction built on top of Go's array type, and so to understand slices we must first understand arrays. - -An array type definition specifies a length and an element type. For example, the type `[4]int` represents an array of four integers. An array's size is fixed; its length is part of its type (`[4]int` and `[5]int` are distinct, incompatible types). Arrays can be indexed in the usual way, so the expression `s[n]` accesses the nth element, starting from zero. - - var a [4]int - a[0] = 1 - i := a[0] - // i == 1 - -Arrays do not need to be initialized explicitly; the zero value of an array is a ready-to-use array whose elements are themselves zeroed: - - // a[2] == 0, the zero value of the int type - -The in-memory representation of `[4]int` is just four integer values laid out sequentially: - -.image go-slices-usage-and-internals_slice-array.png - -Go's arrays are values. An array variable denotes the entire array; it is not a pointer to the first array element (as would be the case in C). This means that when you assign or pass around an array value you will make a copy of its contents. (To avoid the copy you could pass a _pointer_ to the array, but then that's a pointer to an array, not an array.) One way to think about arrays is as a sort of struct but with indexed rather than named fields: a fixed-size composite value. - -An array literal can be specified like so: - - b := [2]string{"Penn", "Teller"} - -Or, you can have the compiler count the array elements for you: - - b := [...]string{"Penn", "Teller"} - -In both cases, the type of `b` is `[2]string`. - -* Slices - -Arrays have their place, but they're a bit inflexible, so you don't see them too often in Go code. Slices, though, are everywhere. They build on arrays to provide great power and convenience. - -The type specification for a slice is `[]T`, where `T` is the type of the elements of the slice. Unlike an array type, a slice type has no specified length. - -A slice literal is declared just like an array literal, except you leave out the element count: - - letters := []string{"a", "b", "c", "d"} - -A slice can be created with the built-in function called `make`, which has the signature, - - func make([]T, len, cap) []T - -where T stands for the element type of the slice to be created. The `make` function takes a type, a length, and an optional capacity. When called, `make` allocates an array and returns a slice that refers to that array. - - var s []byte - s = make([]byte, 5, 5) - // s == []byte{0, 0, 0, 0, 0} - -When the capacity argument is omitted, it defaults to the specified length. Here's a more succinct version of the same code: - - s := make([]byte, 5) - -The length and capacity of a slice can be inspected using the built-in `len` and `cap` functions. - - len(s) == 5 - cap(s) == 5 - -The next two sections discuss the relationship between length and capacity. - -The zero value of a slice is `nil`. The `len` and `cap` functions will both return 0 for a nil slice. - -A slice can also be formed by "slicing" an existing slice or array. Slicing is done by specifying a half-open range with two indices separated by a colon. For example, the expression `b[1:4]` creates a slice including elements 1 through 3 of `b` (the indices of the resulting slice will be 0 through 2). - - b := []byte{'g', 'o', 'l', 'a', 'n', 'g'} - // b[1:4] == []byte{'o', 'l', 'a'}, sharing the same storage as b - -The start and end indices of a slice expression are optional; they default to zero and the slice's length respectively: - - // b[:2] == []byte{'g', 'o'} - // b[2:] == []byte{'l', 'a', 'n', 'g'} - // b[:] == b - -This is also the syntax to create a slice given an array: - - x := [3]string{"Лайка", "Белка", "Стрелка"} - s := x[:] // a slice referencing the storage of x - -* Slice internals - -A slice is a descriptor of an array segment. It consists of a pointer to the array, the length of the segment, and its capacity (the maximum length of the segment). - -.image go-slices-usage-and-internals_slice-struct.png - -Our variable `s`, created earlier by `make([]byte,`5)`, is structured like this: - -.image go-slices-usage-and-internals_slice-1.png - -The length is the number of elements referred to by the slice. The capacity is the number of elements in the underlying array (beginning at the element referred to by the slice pointer). The distinction between length and capacity will be made clear as we walk through the next few examples. - -As we slice `s`, observe the changes in the slice data structure and their relation to the underlying array: - - s = s[2:4] - -.image go-slices-usage-and-internals_slice-2.png - -Slicing does not copy the slice's data. It creates a new slice value that points to the original array. This makes slice operations as efficient as manipulating array indices. Therefore, modifying the _elements_ (not the slice itself) of a re-slice modifies the elements of the original slice: - - d := []byte{'r', 'o', 'a', 'd'} - e := d[2:] - // e == []byte{'a', 'd'} - e[1] = 'm' - // e == []byte{'a', 'm'} - // d == []byte{'r', 'o', 'a', 'm'} - -Earlier we sliced `s` to a length shorter than its capacity. We can grow s to its capacity by slicing it again: - - s = s[:cap(s)] - -.image go-slices-usage-and-internals_slice-3.png - -A slice cannot be grown beyond its capacity. Attempting to do so will cause a runtime panic, just as when indexing outside the bounds of a slice or array. Similarly, slices cannot be re-sliced below zero to access earlier elements in the array. - -* Growing slices (the copy and append functions) - -To increase the capacity of a slice one must create a new, larger slice and copy the contents of the original slice into it. This technique is how dynamic array implementations from other languages work behind the scenes. The next example doubles the capacity of `s` by making a new slice, `t`, copying the contents of `s` into `t`, and then assigning the slice value `t` to `s`: - - t := make([]byte, len(s), (cap(s)+1)*2) // +1 in case cap(s) == 0 - for i := range s { - t[i] = s[i] - } - s = t - -The looping piece of this common operation is made easier by the built-in copy function. As the name suggests, copy copies data from a source slice to a destination slice. It returns the number of elements copied. - - func copy(dst, src []T) int - -The `copy` function supports copying between slices of different lengths (it will copy only up to the smaller number of elements). In addition, `copy` can handle source and destination slices that share the same underlying array, handling overlapping slices correctly. - -Using `copy`, we can simplify the code snippet above: - - t := make([]byte, len(s), (cap(s)+1)*2) - copy(t, s) - s = t - -A common operation is to append data to the end of a slice. This function appends byte elements to a slice of bytes, growing the slice if necessary, and returns the updated slice value: - - func AppendByte(slice []byte, data ...byte) []byte { - m := len(slice) - n := m + len(data) - if n > cap(slice) { // if necessary, reallocate - // allocate double what's needed, for future growth. - newSlice := make([]byte, (n+1)*2) - copy(newSlice, slice) - slice = newSlice - } - slice = slice[0:n] - copy(slice[m:n], data) - return slice - } - -One could use `AppendByte` like this: - - p := []byte{2, 3, 5} - p = AppendByte(p, 7, 11, 13) - // p == []byte{2, 3, 5, 7, 11, 13} - -Functions like `AppendByte` are useful because they offer complete control over the way the slice is grown. Depending on the characteristics of the program, it may be desirable to allocate in smaller or larger chunks, or to put a ceiling on the size of a reallocation. - -But most programs don't need complete control, so Go provides a built-in `append` function that's good for most purposes; it has the signature - - func append(s []T, x ...T) []T - -The `append` function appends the elements `x` to the end of the slice `s`, and grows the slice if a greater capacity is needed. - - a := make([]int, 1) - // a == []int{0} - a = append(a, 1, 2, 3) - // a == []int{0, 1, 2, 3} - -To append one slice to another, use `...` to expand the second argument to a list of arguments. - - a := []string{"John", "Paul"} - b := []string{"George", "Ringo", "Pete"} - a = append(a, b...) // equivalent to "append(a, b[0], b[1], b[2])" - // a == []string{"John", "Paul", "George", "Ringo", "Pete"} - -Since the zero value of a slice (`nil`) acts like a zero-length slice, you can declare a slice variable and then append to it in a loop: - - // Filter returns a new slice holding only - // the elements of s that satisfy f() - func Filter(s []int, fn func(int) bool) []int { - var p []int // == nil - for _, v := range s { - if fn(v) { - p = append(p, v) - } - } - return p - } - -* A possible "gotcha" - -As mentioned earlier, re-slicing a slice doesn't make a copy of the underlying array. The full array will be kept in memory until it is no longer referenced. Occasionally this can cause the program to hold all the data in memory when only a small piece of it is needed. - -For example, this `FindDigits` function loads a file into memory and searches it for the first group of consecutive numeric digits, returning them as a new slice. - - var digitRegexp = regexp.MustCompile("[0-9]+") - - func FindDigits(filename string) []byte { - b, _ := ioutil.ReadFile(filename) - return digitRegexp.Find(b) - } - -This code behaves as advertised, but the returned `[]byte` points into an array containing the entire file. Since the slice references the original array, as long as the slice is kept around the garbage collector can't release the array; the few useful bytes of the file keep the entire contents in memory. - -To fix this problem one can copy the interesting data to a new slice before returning it: - - func CopyDigits(filename string) []byte { - b, _ := ioutil.ReadFile(filename) - b = digitRegexp.Find(b) - c := make([]byte, len(b)) - copy(c, b) - return c - } - -A more concise version of this function could be constructed by using `append`. This is left as an exercise for the reader. - -* Further Reading - -[[http://golang.org/doc/effective_go.html][Effective Go]] contains an in-depth treatment of [[http://golang.org/doc/effective_go.html#slices][slices]] and [[http://golang.org/doc/effective_go.html#arrays][arrays]], and the Go [[http://golang.org/doc/go_spec.html][language specification]] defines [[http://golang.org/doc/go_spec.html#Slice_types][slices]] and their [[http://golang.org/doc/go_spec.html#Length_and_capacity][associated]] [[http://golang.org/doc/go_spec.html#Making_slices_maps_and_channels][helper]] [[http://golang.org/doc/go_spec.html#Appending_and_copying_slices][functions]]. diff --git a/content/go-slices-usage-and-internals_slice-1.png b/content/go-slices-usage-and-internals_slice-1.png deleted file mode 100644 index ba465cf7..00000000 Binary files a/content/go-slices-usage-and-internals_slice-1.png and /dev/null differ diff --git a/content/go-slices-usage-and-internals_slice-2.png b/content/go-slices-usage-and-internals_slice-2.png deleted file mode 100644 index a57581e8..00000000 Binary files a/content/go-slices-usage-and-internals_slice-2.png and /dev/null differ diff --git a/content/go-slices-usage-and-internals_slice-3.png b/content/go-slices-usage-and-internals_slice-3.png deleted file mode 100644 index 64ece5e8..00000000 Binary files a/content/go-slices-usage-and-internals_slice-3.png and /dev/null differ diff --git a/content/go-slices-usage-and-internals_slice-array.png b/content/go-slices-usage-and-internals_slice-array.png deleted file mode 100644 index a533702c..00000000 Binary files a/content/go-slices-usage-and-internals_slice-array.png and /dev/null differ diff --git a/content/go-slices-usage-and-internals_slice-struct.png b/content/go-slices-usage-and-internals_slice-struct.png deleted file mode 100644 index f9141fc5..00000000 Binary files a/content/go-slices-usage-and-internals_slice-struct.png and /dev/null differ diff --git a/content/go-turns-three.article b/content/go-turns-three.article deleted file mode 100644 index bc86875e..00000000 --- a/content/go-turns-three.article +++ /dev/null @@ -1,76 +0,0 @@ -Go turns three -10 Nov 2012 -Tags: community, birthday - -Russ Cox - -* Introduction - -The Go open source project is -[[http://google-opensource.blogspot.com/2009/11/hey-ho-lets-go.html][three years old today]]. - -It's great to look at how far Go has come in those three years. -When we launched, Go was an idea backed by two implementations that worked on Linux and OS X. -The syntax, semantics, and libraries changed regularly as we reacted to feedback from users -and experience with the language. - -Since the open source launch, -we've been joined by -[[http://tip.golang.org/CONTRIBUTORS][hundreds of external contributors]], -who have extended and improved Go in myriad ways, -including writing a Windows port from scratch. -We added a package management system -[[https://groups.google.com/d/msg/golang-nuts/8JFwR3ESjjI/cy7qZzN7Lw4J][goinstall]], -which eventually became the -[[http://golang.org/cmd/go/][go command]]. -We also added -[[http://blog.golang.org/2011/07/go-for-app-engine-is-now-generally.html][support for Go on App Engine]]. -Over the past year we've also given [[http://golang.org/doc/#talks][many talks]], created an [[http://tour.golang.org/][interactive introductory tour]] -and recently we added support for [[http://golang.org/pkg/strings/#pkg-examples][executable examples in package documentation]]. - -Perhaps the most important development in the past year -was the launch of the first stable version, -[[http://blog.golang.org/2012/03/go-version-1-is-released.html][Go 1]]. -People who write Go 1 programs can now be confident that their programs will -continue to compile and run without change, in many environments, -on a time scale of years. -As part of the Go 1 launch we spent months cleaning up the -[[http://golang.org/doc/go1.html][language and libraries]] -to make it something that will age well. - -We're working now toward the release of Go 1.1 in 2013. There will be some -new functionality, but that release will focus primarily on making Go perform -even better than it does today. - -We're especially happy about the community that has grown around Go: -the mailing list and IRC channels seem like they are overflowing with discussion, -and a handful of Go books were published this year. The community is thriving. -Use of Go in production environments has also taken off, especially since Go 1. - -We use Go at Google in a variety of ways, many of them invisible to the outside world. -A few visible ones include -[[https://groups.google.com/d/msg/golang-nuts/BNUNbKSypE0/E4qSfpx9qI8J][serving Chrome and other downloads]], -[[http://code.google.com/p/vitess/][scaling MySQL database at YouTube]], -and of course running the -[[http://golang.org/][Go home page]] -on [[https://developers.google.com/appengine/docs/go/overview][App Engine]]. -Last year's -[[http://blog.golang.org/2011/12/from-zero-to-go-launching-on-google.html][Thanksgiving Doodle]] -and the recent -[[http://www.jamwithchrome.com/technology][Jam with Chrome]] -site are also served by Go programs. - -Other companies and projects are using Go too, including -[[http://www.quora.com/Go-programming-language/Is-Google-Go-ready-for-production-use/answer/Kunal-Anand][BBC Worldwide]], -[[http://dave.cheney.net/wp-content/uploads/2012/08/august-go-meetup.pdf][Canonical]], -[[http://blog.cloudflare.com/go-at-cloudflare][CloudFlare]], -[[http://blog.golang.org/2011/04/go-at-heroku.html][Heroku]], -[[https://plus.google.com/114945221884326152379/posts/d1SVaqkRyTL][Novartis]], -[[http://backstage.soundcloud.com/2012/07/go-at-soundcloud/][SoundCloud]], -[[http://sorcery.smugmug.com/2012/04/06/deriving-json-types-in-go/][SmugMug]], -[[http://blog.golang.org/2011/12/building-stathat-with-go.html][StatHat]], -[[https://tinkercad.com/about/jobs][Tinkercad]], -and -[[https://golang.org/wiki/GoUsers][many others]]. - -Here's to many more years of productive programming in Go. diff --git a/content/go-updates-in-app-engine-171.article b/content/go-updates-in-app-engine-171.article deleted file mode 100644 index 24482e2f..00000000 --- a/content/go-updates-in-app-engine-171.article +++ /dev/null @@ -1,21 +0,0 @@ -Go updates in App Engine 1.7.1 -22 Aug 2012 -Tags: appengine, release - -Andrew Gerrand - -* Introduction - -This week we released version 1.7.1 of the App Engine SDK. It includes some significant updates specific to the App Engine runtime for Go. - -The [[https://developers.google.com/appengine/docs/go/memcache/reference][memcache package]] has had some additions to its [[https://developers.google.com/appengine/docs/go/memcache/reference#Codec][Codec]] convenience type. The SetMulti, AddMulti, CompareAndSwap, and CompareAndSwapMulti methods make it easier to store and update encoded data in the [[https://developers.google.com/appengine/docs/go/memcache/overview][Memcache Service]]. - -The [[https://developers.google.com/appengine/docs/go/tools/uploadingdata][bulkloader tool]] can now be used with Go apps, allowing users to upload and download datastore records in bulk. This is useful for backups and offline processing, and a great help when migrating Python or Java apps to the Go runtime. - -The [[https://developers.google.com/appengine/docs/go/images/overview][Images Service]] is now available to Go users. The new [[https://developers.google.com/appengine/docs/go/images/reference][appengine/image package]] supports serving images directly from Blobstore and resizing or cropping those images on the fly. Note that this is not the full image service as provided by the Python and Java SDKs, as much of the equivalent functionality is available in the [[http://golang.org/pkg/image/][standard Go image package]] and external packages such as [[http://code.google.com/p/graphics-go/][graphics-go]]. - -The new [[https://developers.google.com/appengine/docs/go/backends/runtime#RunInBackground][runtime.RunInBackground]] function allows backend requests to spawn a new request independent of the initial request. These can run in the background as long as the backend stays alive. - -Finally, we have filled in some missing functionality: the [[https://developers.google.com/appengine/docs/go/xmpp/reference][xmpp package]] now supports sending presence updates and chat invitations and retrieving the presence state of another user, and the [[https://developers.google.com/appengine/docs/go/users/reference][user package]] supports authenticating clients with OAuth. - -You can grab the new SDK from the [[https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Go][App Engine downloads page]] and browse the [[https://developers.google.com/appengine/docs/go][updated documentation]]. diff --git a/content/go-version-1-is-released.article b/content/go-version-1-is-released.article deleted file mode 100644 index acf6a319..00000000 --- a/content/go-version-1-is-released.article +++ /dev/null @@ -1,28 +0,0 @@ -Go version 1 is released -28 Mar 2012 -Tags: release, go1 - -Andrew Gerrand - -* Introduction - -.image go-version-1-is-released_gophermega.jpg -Today marks a major milestone in the development of the Go programming language. We're announcing Go version 1, or Go 1 for short, which defines a language and a set of core libraries to provide a stable foundation for creating reliable products, projects, and publications. - -Go 1 is the first release of Go that is available in supported binary distributions. They are available for Linux, FreeBSD, Mac OS X and, we are thrilled to announce, Windows. - -The driving motivation for Go 1 is stability for its users. People who write Go 1 programs can be confident that those programs will continue to compile and run without change, in many environments, on a time scale of years. Similarly, authors who write books about Go 1 can be sure that their examples and explanations will be helpful to readers today and into the future. - -Forward compatibility is part of stability. Code that compiles in Go 1 should, with few exceptions, continue to compile and run throughout the lifetime of that version, even as we issue updates and bug fixes such as Go version 1.1, 1.2, and so on. The [[http://golang.org/doc/go1compat.html][Go 1 compatibility document]] explains the compatibility guidelines in more detail. - -Go 1 is a representation of Go as it is used today, not a major redesign. In its planning, we focused on cleaning up problems and inconsistencies and improving portability. There had long been many changes to Go that we had designed and prototyped but not released because they were backwards-incompatible. Go 1 incorporates these changes, which provide significant improvements to the language and libraries but sometimes introduce incompatibilities for old programs. Fortunately, the [[http://golang.org/cmd/go/#Run_go_tool_fix_on_packages][go fix]] tool can automate much of the work needed to bring programs up to the Go 1 standard. - -Go 1 introduces changes to the language (such as new types for [[http://golang.org/doc/go1.html#rune][Unicode characters]] and [[http://golang.org/doc/go1.html#errors][errors]]) and the standard library (such as the new [[http://golang.org/doc/go1.html#time][time package]] and renamings in the [[http://golang.org/doc/go1.html#strconv][strconv package]]). Also, the package hierarchy has been rearranged to group related items together, such as moving the networking facilities, for instance the [[http://golang.org/pkg/net/rpc/][rpc package]], into subdirectories of net. A complete list of changes is documented in the [[http://golang.org/doc/go1.html][Go 1 release notes]]. That document is an essential reference for programmers migrating code from earlier versions of Go. - -We also restructured the Go tool suite around the new [[http://golang.org/doc/go1.html#cmd_go][go command]], a program for fetching, building, installing and maintaining Go code. The go command eliminates the need for Makefiles to write Go code because it uses the Go program source itself to derive the build instructions. No more build scripts! - -Finally, the release of Go 1 triggers a new release of the [[https://developers.google.com/appengine/docs/go][Google App Engine SDK]]. A similar process of revision and stabilization has been applied to the App Engine libraries, providing a base for developers to build programs for App Engine that will run for years. - -Go 1 is the result of a major effort by the core Go team and our many contributors from the open source community. We thank everyone who helped make this happen. - -There has never been a better time to be a Go programmer. Everything you need to get started is at [[http://golang.org/][golang.org]]. diff --git a/content/go-version-1-is-released_gophermega.jpg b/content/go-version-1-is-released_gophermega.jpg deleted file mode 100644 index 7f07bbd3..00000000 Binary files a/content/go-version-1-is-released_gophermega.jpg and /dev/null differ diff --git a/content/go-videos-from-google-io-2012.article b/content/go-videos-from-google-io-2012.article deleted file mode 100644 index d24b8680..00000000 --- a/content/go-videos-from-google-io-2012.article +++ /dev/null @@ -1,35 +0,0 @@ -Go videos from Google I/O 2012 -2 Jul 2012 -Tags: talk, video, concurrency, appengine, ethos - -Andrew Gerrand - -* Introduction - -Phew! Google I/O is over for another year, and what an event it was. Thanks to our guest speakers and everyone who attended the four Go sessions. It was a lot of fun. - -Here are the session videos: - -*Go*concurrency*patterns* by Rob Pike - -_Concurrency_is_the_key_to_designing_high_performance_network_services._Go's_concurrency_primitives_(goroutines_and_channels)_provide_a_simple_and_efficient_means_of_expressing_concurrent_execution._In_this_talk_we_see_how_tricky_concurrency_problems_can_be_solved_gracefully_with_simple_Go_code._ - -.iframe http://www.youtube.com/embed/f6kdp27TYZs 304 540 - -* Go in production - -_Since_Go's_release_in_2009_many_companies_(besides_Google,_of_course)_have_used_the_language_to_build_cool_stuff._In_this_session_Gustavo_Niemeyer_(Canonical),_Keith_Rarick_(Heroku),_Evan_Shaw_(Iron.io),_and_Patrick_Crosby_(StatHat)_share_their_first-hand_experience_using_Go_in_production_environments._ - -.iframe http://www.youtube.com/embed/kKQLhGZVN4A 304 540 - -* Meet the Go team - -_A_panel_discussion_with_David_Symonds,_Robert_Griesemer,_Rob_Pike,_Ken_Thompson,_Andrew_Gerrand,_and_Brad_Fitzpatrick._ - -.iframe http://www.youtube.com/embed/sln-gJaURzk 304 540 - -*Computing*Map*Tiles*with*Go*on*App*Engine* by Chris Broadfoot and Andrew Gerrand - -_In_this_talk_we_use_the_Maps_API_and_Go_on_App_Engine_to_build_an_app_to_build_custom_tile_sets_for_Google_Maps._The_app_demonstrates_using_Go's_suitability_for_computation_in_the_cloud_and_App_Engine's_key_scalability_features,_such_as_Task_Queues_and_Backends._ - -.iframe http://www.youtube.com/embed/sPagpg0b7no 304 540 diff --git a/content/go-whats-new-in-march-2010.article b/content/go-whats-new-in-march-2010.article deleted file mode 100644 index 858660d2..00000000 --- a/content/go-whats-new-in-march-2010.article +++ /dev/null @@ -1,24 +0,0 @@ -Go: What's New in March 2010 -18 Mar 2010 - -Andrew Gerrand - -* Introduction - -Welcome to the official Go Blog. We, the Go team, hope to use this blog to keep the world up-to-date on the development of the Go programming language and the growing ecosystem of libraries and applications surrounding it. - -It's been a few months since we launched (November last year), so let's talk about what's been happening in Go World since then. - -The core team at Google has continued to develop the language, compilers, packages, tools, and documentation. The compilers now produce code that is in some cases between 2x and an order of magnitude faster than at release. We have put together some graphs of a selection of [[http://godashboard.appspot.com/benchmarks][Benchmarks]], and the the [[http://godashboard.appspot.com/][Build Status]] page tracks the reliability of each changeset submitted to the repository. - -We have made syntax changes to make the language more concise, regular, and flexible. Semicolons have been [[http://groups.google.com/group/golang-nuts/t/5ee32b588d10f2e9][almost entirely removed]] from the language. The [[http://golang.org/doc/go_spec.html#Function_types][...T syntax]] makes it simpler to handle an arbitrary number of typed function parameters. The syntax x[lo:] is now shorthand for x[lo:len(x)]. Go also now natively supports complex numbers. See the [[http://golang.org/doc/devel/release.html][release notes]] for more. - -[[http://golang.org/cmd/godoc/][Godoc]] now provides better support for third-party libraries, and a new tool - [[http://golang.org/cmd/goinstall][goinstall]] - has been released to make it easy to install them. Additionally, we've started working on a package tracking system to make it easier to find what you need. You can view the beginnings of this on the [[http://godashboard.appspot.com/package][Packages page]]. - -More than 40,000 lines of code have been added to [[http://golang.org/pkg/][the standard library]], including many entirely new packages, a sizable portion written by external contributors. - -Speaking of third parties, since launch a vibrant community has flourished on our [[http://groups.google.com/group/golang-nuts/][mailing list]] and irc channel (#go-nuts on freenode). We have officially added [[http://golang.org/CONTRIBUTORS][more than 50 people]] to the project. Their contributions range from bug fixes and documentation corrections to core packages and support for additional operating systems (Go is now supported under FreeBSD, and a [[http://code.google.com/p/go/wiki/WindowsPort][Windows port]] is underway). We regard these community contributions our greatest success so far. - -We've received some good reviews, too. This [[http://www.pcworld.idg.com.au/article/337773/google_go_captures_developers_imaginations/][recent article in PC World]] summarized the enthusiasm surrounding the project. Several bloggers have begun documenting their experiences in the language (see [[http://golang.tumblr.com/][here]], [[http://www.infi.nl/blog/view/id/47][here]], and [[http://freecella.blogspot.com/2010/01/gospecify-basic-setup-of-projects.html][here]] for example) The general reaction of our users has been very positive; one first-timer remarked [[https://groups.google.com/group/golang-nuts/browse_thread/thread/5fabdd59f8562ed2]["I came away extremely impressed. Go walks an elegant line between simplicity and power."]] - -As to the future: we have listened to the myriad voices telling us what they need, and are now focused on getting Go ready for the prime time. We are improving the garbage collector, runtime scheduler, tools, and standard libraries, as well as exploring new language features. 2010 will be an exciting year for Go, and we look forward to collaborating with the community to make it a successful one. diff --git a/content/go-wins-2010-bossie-award.article b/content/go-wins-2010-bossie-award.article deleted file mode 100644 index a48fb1c7..00000000 --- a/content/go-wins-2010-bossie-award.article +++ /dev/null @@ -1,10 +0,0 @@ -Go Wins 2010 Bossie Award -6 Sep 2010 - -Andrew Gerrand - -* Introduction - -The Go project has been awarded a [[http://www.infoworld.com/d/open-source/bossie-awards-2010-the-best-open-source-application-development-software-140¤t=2&last=1][Bossie Award]] for "best open source application development software." The [[http://www.infoworld.com/d/open-source/bossie-awards-2010-the-best-open-source-software-the-year-115][Bossie Awards]] have been given each year since 2007 by [[http://infoworld.com/][InfoWorld]] to recognize the best Open Source projects. Their citation reads: - -Google's Go language tries to restore simplicity to programming. It does away with numerous constructs that have crept into all OO languages by re-imagining how to simplify and improve the conversation between a developer and the code. Go provides garbage collection, type safety, memory safety, and built-in support for concurrency and for Unicode characters. In addition, it compiles (fast!) to binaries for multiple platforms. Go is still in development (new features are being added) and it has limitations, notably poor support for Windows. But it shows a new, exciting direction in programming languages.The Go team are pleased to accept this award. It reflects not only our hard work, but also the efforts of the growing community of Go developers to whom we must extend our sincere thanks. diff --git a/content/go1.3.article b/content/go1.3.article deleted file mode 100644 index 17dc7b72..00000000 --- a/content/go1.3.article +++ /dev/null @@ -1,57 +0,0 @@ -Go 1.3 is released -18 Jun 2014 - -Andrew Gerrand - -* Go 1.3 is released - -Today we are happy to announce the release of [[http://golang.org/doc/go1.3][Go 1.3]]. -This release comes six months after our last major release and provides better -performance, improved tools, support for running Go in new environments, and more. -All Go users should upgrade to Go 1.3. -You can grab the release from our [[http://golang.org/dl/][downloads page]] and -find the full list of improvements and fixes in the -[[http://golang.org/doc/go1.3][release notes]]. -What follows are some highlights. - -[[http://godoc.org/code.google.com/p/go.tools/cmd/godoc][Godoc]], -the Go documentation server, now performs static analysis. -When enabled with the -analysis flag, analysis results are presented -in both the source and package documentation views, making it easier -than ever to navigate and understand Go programs. -See [[http://golang.org/lib/godoc/analysis/help.html][the documentation]] for the details. - -The gc toolchain now supports the Native Client (NaCl) execution sandbox on the -32- and 64-bit Intel architectures. -This permits the safe execution of untrusted code, useful in environments such as the -[[http://blog.golang.org/playground][Playground]]. -To set up NaCl on your system see the [[http://golang.org/wiki/NativeClient][NativeClient wiki page]]. - -Also included in this release is experimental support for the DragonFly BSD, -Plan 9, and Solaris operating systems. To use Go on these systems you must -[[http://golang.org/doc/install/source][install from source]]. - -Changes to the runtime have improved the -[[http://golang.org/doc/go1.3#performance][performance]] of Go binaries, -with an improved garbage collector, a new -[[http://golang.org/s/contigstacks]["contiguous" goroutine stack management strategy]], -a faster race detector, and improvements to the regular expression engine. - -As part of the general [[http://golang.org/s/go13linker][overhaul]] of the Go -linker, the compilers and linkers have been refactored. The instruction -selection phase that was part of the linker has been moved to the compiler. -This can speed up incremental builds for large projects. - -The [[http://golang.org/doc/go1.3#garbage_collector][garbage collector]] is now -precise when examining stacks (collection of the heap has been precise since Go -1.1), meaning that a non-pointer value such as an integer will never be -mistaken for a pointer and prevent unused memory from being reclaimed. This -change affects code that uses package unsafe; if you have unsafe code you -should read the [[http://golang.org/doc/go1.3#garbage_collector][release notes]] -carefully to see if your code needs updating. - -We would like to thank the many people who contributed to this release; -it would not have been possible without your help. - -So, what are you waiting for? -Head on over to the [[http://golang.org/dl/][downloads page]] and start hacking. diff --git a/content/go1.4.article b/content/go1.4.article deleted file mode 100644 index d106b356..00000000 --- a/content/go1.4.article +++ /dev/null @@ -1,64 +0,0 @@ -Go 1.4 is released -10 Dec 2014 - -Andrew Gerrand -adg@golang.org - -* Introduction - -Today we announce Go 1.4, the fifth major stable release of Go, arriving six -months after our previous major release [[https://blog.golang.org/go1.3][Go 1.3]]. -It contains a small language change, support for more operating systems -and processor architectures, and improvements to the tool chain and libraries. -As always, Go 1.4 keeps the promise of compatibility, and almost everything -will continue to compile and run without change when moved to 1.4. -For the full details, see the [[https://golang.org/doc/go1.4][Go 1.4 release notes]]. - -The most notable new feature in this release is official support for Android. -Using the support in the core and the libraries in the -[[https://godoc.org/golang.org/x/mobile][golang.org/x/mobile]] repository, -it is now possible to write simple Android apps using only Go code. -At this stage, the support libraries are still nascent and under heavy development. -Early adopters should expect a bumpy ride, but we welcome the community to get involved. - -The language change is a tweak to the syntax of for-range loops. -You may now write "for range s {" to loop over each item from s, -without having to assign the value, loop index, or map key. -See the [[https://golang.org/doc/go1.4#forrange][release notes]] for details. - -The go command has a new subcommand, go generate, to automate the running of -tools to generate source code before compilation. -For example, it can be used to automate the generation of String methods for -typed constants using the -[[https://godoc.org/golang.org/x/tools/cmd/stringer/][new stringer tool]]. -For more information, see the [[https://golang.org/s/go1.4-generate][design document]]. - -Most programs will run about the same speed or slightly faster in 1.4 than in -1.3; some will be slightly slower. -There are many changes, making it hard to be precise about what to expect. -See the [[https://golang.org/doc/go1.4#performance][release notes]] for more discussion. - -And, of course, there are many more improvements and bug fixes. - -In case you missed it, a few weeks ago the sub-repositories were moved to new locations. -For example, the go.tools packages are now imported from "golang.org/x/tools". -See the [[https://groups.google.com/d/msg/golang-announce/eD8dh3T9yyA/HDOEU_ZSmvAJ][announcement post]] for details. - -This release also coincides with the project's move from Mercurial to Git (for -source control), Rietveld to Gerrit (for code review), and Google Code to -Github (for issue tracking and wiki). -The move affects the core Go repository and its sub-repositories. -You can find the canonical Git repositories at -[[https://go.googlesource.com][go.googlesource.com]], -and the issue tracker and wiki at the -[[https://github.com/golang/go][golang/go GitHub repo]]. - -While development has already moved over to the new infrastructure, -for the 1.4 release we still recommend that users who -[[https://golang.org/doc/install/source][install from source]] -use the Mercurial repositories. - -For App Engine users, Go 1.4 is now available for beta testing. -See [[https://groups.google.com/d/msg/google-appengine-go/ndtQokV3oFo/25wV1W9JtywJ][the announcement]] for details. - -From all of us on the Go team, please enjoy Go 1.4, and have a happy holiday season. diff --git a/content/go1.5.article b/content/go1.5.article deleted file mode 100644 index 4abff055..00000000 --- a/content/go1.5.article +++ /dev/null @@ -1,77 +0,0 @@ -Go 1.5 is released -19 Aug 2015 - -Andrew Gerrand -adg@golang.org - -* Introduction - -Today the Go project is proud to release Go 1.5, -the sixth major stable release of Go. - -This release includes significant changes to the implementation. -The compiler tool chain was [[https://golang.org/doc/go1.5#c][translated from C to Go]], -removing the last vestiges of C code from the Go code base. -The garbage collector was [[https://golang.org/doc/go1.5#gc][completely redesigned]], -yielding a [[https://talks.golang.org/2015/go-gc.pdf][dramatic reduction]] -in garbage collection pause times. -Related improvements to the scheduler allowed us to change the default -[[https://golang.org/pkg/runtime/#GOMAXPROCS][GOMAXPROCS]] value -(the number of concurrently executing goroutines) -from 1 to the number of logical CPUs. -Changes to the linker enable distributing Go packages as shared libraries to -link into Go programs, and building Go packages into archives or shared -libraries that may be linked into or loaded by C programs -([[https://golang.org/s/execmodes][design doc]]). - -The release also includes [[https://golang.org/doc/go1.5#go_command][improvements to the developer tools]]. -Support for [[https://golang.org/s/go14internal]["internal" packages]] -permits sharing implementation details between packages. -[[https://golang.org/s/go15vendor][Experimental support]] for "vendoring" -external dependencies is a step toward a standard mechanism for managing -dependencies in Go programs. -The new "[[https://golang.org/cmd/trace/][go tool trace]]" command enables the -visualisation of  program traces generated by new tracing infrastructure in the -runtime. -The new "[[https://golang.org/cmd/go/#hdr-Show_documentation_for_package_or_symbol][go doc]]" -command provides an improved command-line interface for viewing Go package documentation. - -There are also several [[https://golang.org/doc/go1.5#ports][new operating system and architecture ports]]. -The more mature new ports are darwin/arm, -darwin/arm64 (Apple's iPhone and iPad devices), -and linux/arm64. -There is also experimental support for ppc64 and ppc64le -(IBM 64-bit PowerPC, big and little endian). - -The new darwin/arm64 port and external linking features fuel the -[[https://godoc.org/golang.org/x/mobile][Go mobile project]], an experiment to -see how Go might be used for building apps on Android and iOS devices. -(The Go mobile work itself is not part of this release.) - -The only language change is very minor, -[[https://golang.org/doc/go1.5#language][the lifting of a restriction in the map literal syntax]] -to make them more succinct and consistent with slice literals. - -The standard library saw many additions and improvements, too. -The flag package now shows [[https://golang.org/doc/go1.5#flag][cleaner usage messages]]. -The math/big package now provides a [[https://golang.org/pkg/math/big/#Float][Float]] -type for computing with arbitrary-precision floating point numbers. -An [[https://golang.org/doc/go1.5#net][improvement]] to the DNS resolver on -Linux and BSD systems has removed the cgo requirement for programs that do name -lookups. -The [[https://golang.org/pkg/go/types/][go/types]] package has been -[[https://golang.org/doc/go1.5#go_types][moved]] to the standard library from -the [[https://godoc.org/golang.org/x/tools][golang.org/x/tools]] repository. -(The new [[https://golang.org/pkg/go/constant/][go/constant]] and -[[https://golang.org/pkg/go/importer/][go/importer]] packages are also a result -of this move.) -The reflect package has added the -[[https://golang.org/pkg/reflect/#ArrayOf][ArrayOf]] and -[[https://golang.org/pkg/reflect/#FuncOf][FuncOf]] functions, analogous to the -existing [[https://golang.org/pkg/reflect/#SliceOf][SliceOf]] function. -And, of course, there is the usual -[[https://golang.org/doc/go1.5#minor_library_changes][list of smaller fixes and improvements]]. - -For the full story, see the [[https://golang.org/doc/go1.5][detailed release notes]]. -Or if you just can't wait to get started, -head over to the [[https://golang.org/dl/][downloads page]] to get Go 1.5 now. diff --git a/content/go12.article b/content/go12.article deleted file mode 100644 index eb997887..00000000 --- a/content/go12.article +++ /dev/null @@ -1,77 +0,0 @@ -Go 1.2 is released -1 Dec 2013 -Tags: release - -Andrew Gerrand - -* Introduction - -We are pleased to announce the release of Go 1.2, the latest stable version of -the Go Programming Language. - -Binary distributions may be downloaded from the -[[http://golang.org/doc/install][usual place]] or if you prefer to -[[http://golang.org/doc/install/source][compile from source]] you should use -the `release` or `go1.2` tags. - -This new release comes nearly seven months after the release of Go 1.1 in May, -a much shorter period than the 14 months between 1.1 and 1.0. -We anticipate a comparable interval between future major releases. - -[[http://golang.org/doc/go1.2][Go 1.2]] includes a couple of minor -language changes, several improvements to the language implementation and -tools, some performance improvements, and many additions and -(backward-compatible) changes to the standard library. - -Please read the [[http://golang.org/doc/go1.2][release notes]] for all -the details, as some changes may affect the behavior of existing (buggy) programs. -What follows is the highlights of the release. - -A new [[http://golang.org/doc/go1.2#three_index][three-index slice syntax]] -adds the ability to specify capacity as well as length. This allows the -programmer to pass a slice value that can only access a limited portion of the -underlying array, a technique that previously required the use of the unsafe -package. - -A major new feature of the tool chain is the facility to compute and display -[[http://golang.org/doc/go1.2#cover][test coverage results]]. -See the [[http://golang.org/cmd/go/#hdr-Description_of_testing_flags][`go`test`]] -and [[http://godoc.org/code.google.com/p/go.tools/cmd/cover/][cover tool]] -documentation for details. Later this week we will publish an article that -discusses this new feature in detail. - -Goroutines are now [[http://golang.org/doc/go1.2#preemption][pre-emptively scheduled]], -in that the scheduler is invoked occasionally upon entry to a function. -This can prevent busy goroutines from starving other goroutines on the same -thread. - -An increase to the default goroutine stack size should improve the -performance of some programs. (The old size had a tendency to introduce -expensive stack-segment switching in performance-critical sections.) -On the other end, new restrictions on -[[http://golang.org/doc/go1.2#stack_size][stack sizes]] and -[[http://golang.org/doc/go1.2#thread_limit][the number of operating system threads]] -should prevent misbehaving programs from consuming all the resources of a -machine. (These limits may be adjusted using new functions in the -[[http://golang.org/pkg/runtime/debug][`runtime/debug` package]].) - -Finally, among the [[http://golang.org/doc/go1.2#library][many changes to the standard library]], -significant changes include -the new [[http://golang.org/doc/go1.2#encoding][`encoding` package]], -[[http://golang.org/doc/go1.2#fmt_indexed_arguments][indexed arguments]] in `Printf` format strings, and -some [[http://golang.org/doc/go1.2#text_template][convenient additions]] to the template packages. - -As part of the release, the [[http://play.golang.org/][Go Playground]] has been -updated to Go 1.2. This also affects services that use the Playground, such as -[[http://tour.golang.org/][the Go Tour]] and this blog. -The update also adds the ability to use threads and the `os`, `net`, and -`unsafe` packages inside the sandbox, making it more like a real Go environment. - -To everyone who helped make this release possible, from the many users who -submitted bug reports to the 116 (!) contributors who committed more than 1600 -changes to the core: Your help is invaluable to the project. Thank you! - -_This_blog_post_is_the_first_of_the_ -[[http://blog.gopheracademy.com/day-01-go-1.2][Go Advent Calendar]], -_a_series_of_daily_articles_presented_by_the_ -[[http://gopheracademy.com/][Gopher Academy]] _from_December_1_to_25._ diff --git a/content/go15gc.article b/content/go15gc.article deleted file mode 100644 index 56f75688..00000000 --- a/content/go15gc.article +++ /dev/null @@ -1,114 +0,0 @@ -Go GC: Prioritizing low latency and simplicity -31 Aug 2015 - -Richard Hudson -rlh@golang.org - -* The Setup - -Go is building a garbage collector (GC) not only for 2015 but for 2025 and -beyond: A GC that supports today’s software development and scales along with -new software and hardware throughout the next decade. Such a future has no -place for stop-the-world GC pauses, which have been an impediment to broader -uses of safe and secure languages such as Go. - -Go 1.5, the first glimpse of this future, achieves GC latencies well below the -10 millisecond goal we set a year ago. We presented some impressive numbers -in [[https://talks.golang.org/2015/go-gc.pdf][a talk at Gophercon]]. -The latency improvements have generated a lot of attention; -Robin Verlangen’s blog post -[[https://medium.com/@robin.verlangen/billions-of-request-per-day-meet-go-1-5-362bfefa0911][_Billions_of_requests_per_day_meet_Go_1.5_]] -validates our direction with end to end results. -We also particularly enjoyed -[[https://twitter.com/inconshreveable/status/620650786662555648][Alan Shreve’s production server graphs]] -and his "Holy 85% reduction" comment. - -Today 16 gigabytes of RAM costs $100 and CPUs come with many cores, each with -multiple hardware threads. In a decade this hardware will seem quaint but the -software being built in Go today will need to scale to meet expanding needs and -the next big thing. Given that hardware will provide the power to increase -throughput, Go’s garbage collector is being designed to favor low latency and -tuning via only a single knob. Go 1.5 is the first big step down this path and -these first steps will forever influence Go and the applications it best -supports. This blog post gives a high-level overview of what we have done for -the Go 1.5 collector. - -* The Embellishment - -To create a garbage collector for the next decade, we turned to an algorithm -from decades ago. Go's new garbage collector is a _concurrent_, _tri-color_, -_mark-sweep_ collector, an idea first proposed by -[[http://dl.acm.org/citation.cfm?id=359655][Dijkstra in 1978]]. -This is a deliberate divergence from most "enterprise" grade garbage collectors -of today, and one that we believe is well suited to the properties of modern -hardware and the latency requirements of modern software. - -In a tri-color collector, every object is either white, grey, or black and we -view the heap as a graph of connected objects. At the start of a GC cycle all -objects are white. The GC visits all _roots_, which are objects directly -accessible by the application such as globals and things on the stack, and -colors these grey. The GC then chooses a grey object, blackens it, and then -scans it for pointers to other objects. When this scan finds a pointer to a -white object, it turns that object grey. This process repeats until there are -no more grey objects. At this point, white objects are known to be unreachable -and can be reused. - -This all happens concurrently with the application, known as the _mutator_, -changing pointers while the collector is running. Hence, the mutator must -maintain the invariant that no black object points to a white object, lest the -garbage collector lose track of an object installed in a part of the heap it -has already visited. Maintaining this invariant is the job of the -_write_barrier_, which is a small function run by the mutator whenever a -pointer in the heap is modified. Go’s write barrier colors the now-reachable -object grey if it is currently white, ensuring that the garbage collector will -eventually scan it for pointers. - -Deciding when the job of finding all grey objects is done is subtle and can be -expensive and complicated if we want to avoid blocking the mutators. To keep -things simple Go 1.5 does as much work as it can concurrently and then briefly -stops the world to inspect all potential sources of grey objects. Finding the -sweet spot between the time needed for this final stop-the-world and the total -amount of work that this GC does is a major deliverable for Go 1.6. - -Of course the devil is in the details. When do we start a GC cycle? What -metrics do we use to make that decision? How should the GC interact with the Go -scheduler? How do we pause a mutator thread long enough to scan its stack? - How do we represent white, grey, and black so we can efficiently find and scan -grey objects? How do we know where the roots are? How do we know where in an -object pointers are located? How do we minimize memory fragmentation? How do we -deal with cache performance issues? How big should the heap be? And on and on, -some related to allocation, some to finding reachable objects, some related to -scheduling, but many related to performance. Low-level discussions of each of -these areas are beyond the scope of this blog post. - -At a higher level, one approach to solving performance problems is to add GC -knobs, one for each performance issue. The programmer can then turn the knobs -in search of appropriate settings for their application. The downside is that -after a decade with one or two new knobs each year you end up with the GC Knobs -Turner Employment Act. Go is not going down that path. Instead we provide a -single knob, called GOGC. This value controls the total size of the heap -relative to the size of reachable objects. The default value of 100 means that -total heap size is now 100% bigger than (i.e., twice) the size of the reachable -objects after the last collection. 200 means total heap size is 200% bigger -than (i.e., three times) the size of the reachable objects. If you want to -lower the total time spent in GC, increase GOGC. If you want to trade more GC -time for less memory, lower GOGC. - -More importantly as RAM doubles with the next generation of hardware, simply -doubling GOGC will halve the number of GC cycles. On the other hand since GOGC -is based on reachable object size, doubling the load by doubling the reachable -objects requires no retuning. The application just scales. -Furthermore, unencumbered by ongoing support for dozens of knobs, the runtime -team can focus on improving the runtime based on feedback from real customer -applications. - -* The Punchline - -Go 1.5’s GC ushers in a future where stop-the-world pauses are no longer a -barrier to moving to a safe and secure language. It is a future where -applications scale effortlessly along with hardware and as hardware becomes -more powerful the GC will not be an impediment to better, more scalable -software. It’s a good place to be for the next decade and beyond. -For more details about the 1.5 GC and how we eliminated latency issues see the -[[https://www.youtube.com/watch?v=aiv1JOfMjm0][Go GC: Latency Problem Solved presentation]] -or [[https://talks.golang.org/2015/go-gc.pdf][the slides]]. diff --git a/content/gobs-of-data.article b/content/gobs-of-data.article deleted file mode 100644 index 95dfa67d..00000000 --- a/content/gobs-of-data.article +++ /dev/null @@ -1,142 +0,0 @@ -Gobs of data -24 Mar 2011 -Tags: gob, json, protobuf, xml, technical - -Rob Pike - -* Introduction - -To transmit a data structure across a network or to store it in a file, it must be encoded and then decoded again. There are many encodings available, of course: [[http://www.json.org/][JSON]], [[http://www.w3.org/XML/][XML]], Google's [[http://code.google.com/p/protobuf][protocol buffers]], and more. And now there's another, provided by Go's [[http://golang.org/pkg/encoding/gob/][gob]] package. - -Why define a new encoding? It's a lot of work and redundant at that. Why not just use one of the existing formats? Well, for one thing, we do! Go has [[http://golang.org/pkg/][packages]] supporting all the encodings just mentioned (the [[http://github.com/golang/protobuf][protocol buffer package]] is in a separate repository but it's one of the most frequently downloaded). And for many purposes, including communicating with tools and systems written in other languages, they're the right choice. - -But for a Go-specific environment, such as communicating between two servers written in Go, there's an opportunity to build something much easier to use and possibly more efficient. - -Gobs work with the language in a way that an externally-defined, language-independent encoding cannot. At the same time, there are lessons to be learned from the existing systems. - -* Goals - -The gob package was designed with a number of goals in mind. - -First, and most obvious, it had to be very easy to use. First, because Go has reflection, there is no need for a separate interface definition language or "protocol compiler". The data structure itself is all the package should need to figure out how to encode and decode it. On the other hand, this approach means that gobs will never work as well with other languages, but that's OK: gobs are unashamedly Go-centric. - -Efficiency is also important. Textual representations, exemplified by XML and JSON, are too slow to put at the center of an efficient communications network. A binary encoding is necessary. - -Gob streams must be self-describing. Each gob stream, read from the beginning, contains sufficient information that the entire stream can be parsed by an agent that knows nothing a priori about its contents. This property means that you will always be able to decode a gob stream stored in a file, even long after you've forgotten what data it represents. - -There were also some things to learn from our experiences with Google protocol buffers. - -* Protocol buffer misfeatures - -Protocol buffers had a major effect on the design of gobs, but have three features that were deliberately avoided. (Leaving aside the property that protocol buffers aren't self-describing: if you don't know the data definition used to encode a protocol buffer, you might not be able to parse it.) - -First, protocol buffers only work on the data type we call a struct in Go. You can't encode an integer or array at the top level, only a struct with fields inside it. That seems a pointless restriction, at least in Go. If all you want to send is an array of integers, why should you have to put it into a struct first? - -Next, a protocol buffer definition may specify that fields `T.x` and `T.y` are required to be present whenever a value of type `T` is encoded or decoded. Although such required fields may seem like a good idea, they are costly to implement because the codec must maintain a separate data structure while encoding and decoding, to be able to report when required fields are missing. They're also a maintenance problem. Over time, one may want to modify the data definition to remove a required field, but that may cause existing clients of the data to crash. It's better not to have them in the encoding at all. (Protocol buffers also have optional fields. But if we don't have required fields, all fields are optional and that's that. There will be more to say about optional fields a little later.) - -The third protocol buffer misfeature is default values. If a protocol buffer omits the value for a "defaulted" field, then the decoded structure behaves as if the field were set to that value. This idea works nicely when you have getter and setter methods to control access to the field, but is harder to handle cleanly when the container is just a plain idiomatic struct. Required fields are also tricky to implement: where does one define the default values, what types do they have (is text UTF-8? uninterpreted bytes? how many bits in a float?) and despite the apparent simplicity, there were a number of complications in their design and implementation for protocol buffers. We decided to leave them out of gobs and fall back to Go's trivial but effective defaulting rule: unless you set something otherwise, it has the "zero value" for that type - and it doesn't need to be transmitted. - -So gobs end up looking like a sort of generalized, simplified protocol buffer. How do they work? - -* Values - -The encoded gob data isn't about types like `int8` and `uint16`. Instead, somewhat analogous to constants in Go, its integer values are abstract, sizeless numbers, either signed or unsigned. When you encode an `int8`, its value is transmitted as an unsized, variable-length integer. When you encode an `int64`, its value is also transmitted as an unsized, variable-length integer. (Signed and unsigned are treated distinctly, but the same unsized-ness applies to unsigned values too.) If both have the value 7, the bits sent on the wire will be identical. When the receiver decodes that value, it puts it into the receiver's variable, which may be of arbitrary integer type. Thus an encoder may send a 7 that came from an `int8`, but the receiver may store it in an `int64`. This is fine: the value is an integer and as a long as it fits, everything works. (If it doesn't fit, an error results.) This decoupling from the size of the variable gives some flexibility to the encoding: we can expand the type of the integer variable as the software evolves, but still be able to decode old data. - -This flexibility also applies to pointers. Before transmission, all pointers are flattened. Values of type `int8`, `*int8`, `**int8`, `****int8`, etc. are all transmitted as an integer value, which may then be stored in `int` of any size, or `*int`, or `******int`, etc. Again, this allows for flexibility. - -Flexibility also happens because, when decoding a struct, only those fields that are sent by the encoder are stored in the destination. Given the value - - type T struct{ X, Y, Z int } // Only exported fields are encoded and decoded. - var t = T{X: 7, Y: 0, Z: 8} - -the encoding of `t` sends only the 7 and 8. Because it's zero, the value of `Y` isn't even sent; there's no need to send a zero value. - -The receiver could instead decode the value into this structure: - - type U struct{ X, Y *int8 } // Note: pointers to int8s - var u U - -and acquire a value of `u` with only `X` set (to the address of an `int8` variable set to 7); the `Z` field is ignored - where would you put it? When decoding structs, fields are matched by name and compatible type, and only fields that exist in both are affected. This simple approach finesses the "optional field" problem: as the type `T` evolves by adding fields, out of date receivers will still function with the part of the type they recognize. Thus gobs provide the important result of optional fields - extensibility - without any additional mechanism or notation. - -From integers we can build all the other types: bytes, strings, arrays, slices, maps, even floats. Floating-point values are represented by their IEEE 754 floating-point bit pattern, stored as an integer, which works fine as long as you know their type, which we always do. By the way, that integer is sent in byte-reversed order because common values of floating-point numbers, such as small integers, have a lot of zeros at the low end that we can avoid transmitting. - -One nice feature of gobs that Go makes possible is that they allow you to define your own encoding by having your type satisfy the [[http://golang.org/pkg/encoding/gob/#GobEncoder][GobEncoder]] and [[http://golang.org/pkg/encoding/gob/#GobDecoder][GobDecoder]] interfaces, in a manner analogous to the [[http://golang.org/pkg/encoding/json/][JSON]] package's [[http://golang.org/pkg/encoding/json/#Marshaler][Marshaler]] and [[http://golang.org/pkg/encoding/json/#Unmarshaler][Unmarshaler]] and also to the [[http://golang.org/pkg/fmt/#Stringer][Stringer]] interface from [[http://golang.org/pkg/fmt/][package fmt]]. This facility makes it possible to represent special features, enforce constraints, or hide secrets when you transmit data. See the [[http://golang.org/pkg/encoding/gob/][documentation]] for details. - -* Types on the wire - -The first time you send a given type, the gob package includes in the data stream a description of that type. In fact, what happens is that the encoder is used to encode, in the standard gob encoding format, an internal struct that describes the type and gives it a unique number. (Basic types, plus the layout of the type description structure, are predefined by the software for bootstrapping.) After the type is described, it can be referenced by its type number. - -Thus when we send our first type `T`, the gob encoder sends a description of `T` and tags it with a type number, say 127. All values, including the first, are then prefixed by that number, so a stream of `T` values looks like: - - ("define type id" 127, definition of type T)(127, T value)(127, T value), ... - -These type numbers make it possible to describe recursive types and send values of those types. Thus gobs can encode types such as trees: - - type Node struct { - Value int - Left, Right *Node - } - -(It's an exercise for the reader to discover how the zero-defaulting rule makes this work, even though gobs don't represent pointers.) - -With the type information, a gob stream is fully self-describing except for the set of bootstrap types, which is a well-defined starting point. - -* Compiling a machine - -The first time you encode a value of a given type, the gob package builds a little interpreted machine specific to that data type. It uses reflection on the type to construct that machine, but once the machine is built it does not depend on reflection. The machine uses package unsafe and some trickery to convert the data into the encoded bytes at high speed. It could use reflection and avoid unsafe, but would be significantly slower. (A similar high-speed approach is taken by the protocol buffer support for Go, whose design was influenced by the implementation of gobs.) Subsequent values of the same type use the already-compiled machine, so they can be encoded right away. - -[Update: As of Go 1.4, package unsafe is no longer use by the gob package, with a modest performance drop.] - -Decoding is similar but harder. When you decode a value, the gob package holds a byte slice representing a value of a given encoder-defined type to decode, plus a Go value into which to decode it. The gob package builds a machine for that pair: the gob type sent on the wire crossed with the Go type provided for decoding. Once that decoding machine is built, though, it's again a reflectionless engine that uses unsafe methods to get maximum speed. - -* Use - -There's a lot going on under the hood, but the result is an efficient, easy-to-use encoding system for transmitting data. Here's a complete example showing differing encoded and decoded types. Note how easy it is to send and receive values; all you need to do is present values and variables to the [[http://golang.org/pkg/encoding/gob/][gob package]] and it does all the work. - - package main - - import ( - "bytes" - "encoding/gob" - "fmt" - "log" - ) - - type P struct { - X, Y, Z int - Name string - } - - type Q struct { - X, Y *int32 - Name string - } - - func main() { - // Initialize the encoder and decoder. Normally enc and dec would be - // bound to network connections and the encoder and decoder would - // run in different processes. - var network bytes.Buffer // Stand-in for a network connection - enc := gob.NewEncoder(&network) // Will write to network. - dec := gob.NewDecoder(&network) // Will read from network. - // Encode (send) the value. - err := enc.Encode(P{3, 4, 5, "Pythagoras"}) - if err != nil { - log.Fatal("encode error:", err) - } - // Decode (receive) the value. - var q Q - err = dec.Decode(&q) - if err != nil { - log.Fatal("decode error:", err) - } - fmt.Printf("%q: {%d,%d}\n", q.Name, *q.X, *q.Y) - } - -You can compile and run this example code in the [[http://play.golang.org/p/_-OJV-rwMq][Go Playground]]. - -The [[http://golang.org/pkg/net/rpc/][rpc package]] builds on gobs to turn this encode/decode automation into transport for method calls across the network. That's a subject for another article. - -* Details - -The [[http://golang.org/pkg/encoding/gob/][gob package documentation]], especially the file [[http://golang.org/src/pkg/encoding/gob/doc.go][doc.go]], expands on many of the details described here and includes a full worked example showing how the encoding represents data. If you are interested in the innards of the gob implementation, that's a good place to start. diff --git a/content/godoc-documenting-go-code.article b/content/godoc-documenting-go-code.article deleted file mode 100644 index eda08607..00000000 --- a/content/godoc-documenting-go-code.article +++ /dev/null @@ -1,52 +0,0 @@ -Godoc: documenting Go code -31 Mar 2011 -Tags: godoc, technical - -Andrew Gerrand - -* Introduction - -The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable. Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers to produce good documentation, the better for everyone. - -To that end, we have developed the [[http://golang.org/cmd/godoc/][godoc]] documentation tool. This article describes godoc's approach to documentation, and explains how you can use our conventions and tools to write good documentation for your own projects. - -Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from a function's [[http://golang.org/pkg/strings/#HasPrefix][documentation]] to its [[http://golang.org/src/pkg/strings/strings.go?#L312][implementation]] with one click. - -Godoc is conceptually related to Python's [[http://www.python.org/dev/peps/pep-0257/][Docstring]] and Java's [[http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html][Javadoc]], but its design is simpler. The comments read by godoc are not language constructs (as with Docstring) nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments, the sort you would want to read even if godoc didn't exist. - -The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside the item it documents. For example, this is the documentation for the `fmt` package's [[http://golang.org/pkg/fmt/#Fprint][`Fprint`]] function: - - // Fprint formats using the default formats for its operands and writes to w. - // Spaces are added between operands when neither is a string. - // It returns the number of bytes written and any write error encountered. - func Fprint(w io.Writer, a ...interface{}) (n int, err error) { - -Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes it read better when tools truncate it for brevity, such as when they extract the first line or sentence. - -Comments on package declarations should provide general package documentation. These comments can be short, like the [[http://golang.org/pkg/sort/][`sort`]] package's brief description: - - // Package sort provides primitives for sorting slices and user-defined - // collections. - package sort - -They can also be detailed like the [[http://golang.org/pkg/encoding/gob/][gob package]]'s overview. That package uses another convention for packages that need large amounts of introductory documentation: the package comment is placed in its own file, [[http://golang.org/src/pkg/encoding/gob/doc.go][doc.go]], which contains only those comments and a package clause. - -When writing package comments of any size, keep in mind that their first sentence will appear in godoc's [[http://golang.org/pkg/][package list]]. - -Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception. Top-level comments that begin with the word `"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who” part should be the user name of someone who could provide more information. For example, this is a known issue from the [[http://golang.org/pkg/bytes/#bugs][bytes package]]: - - // BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly. - -There are a few formatting rules that Godoc uses when converting comments to HTML: - -- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs. - -- Pre-formatted text must be indented relative to the surrounding comment text (see gob's [[http://golang.org/src/pkg/encoding/gob/doc.go][doc.go]] for an example). - -- URLs will be converted to HTML links; no special markup is necessary. - -Note that none of these rules requires you to do anything out of the ordinary. - -In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including all of the standard library, already follows the conventions. - -Your own code can present good documentation just by having comments as described above. Any Go packages installed inside `$GOROOT/src/pkg` and any `GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify additional paths for indexing via the `-path` flag or just by running `"godoc`."` in the source directory. See the [[http://golang.org/cmd/godoc/][godoc documentation]] for more details. diff --git a/content/gopher.article b/content/gopher.article deleted file mode 100644 index dca0307c..00000000 --- a/content/gopher.article +++ /dev/null @@ -1,67 +0,0 @@ -The Go Gopher -24 Mar 2014 -Tags: gopher - -Rob Pike - -Andrew Gerrand - -* The Go Gopher - -.image gopher/header.jpg - -The Go gopher is an iconic mascot and one of the most distinctive features of the Go project. In this post we'll talk about its origins, evolution, and behavior. - -About 15 years ago—long before the Go project—the gopher first appeared as a promotion for the [[https://wfmu.org/][WFMU radio station]] in New Jersey. [[http://reneefrench.blogspot.com][Renee French]] was commissioned to design a T-shirt for an annual fundraiser and out came the gopher. - -.image gopher/wfmu.jpg - -The gopher next made an appearance at Bell Labs, as Bob Flandrena's [[http://research.swtch.com/face][avatar]] in the Bell Labs mail system. Other Renee drawings became avatars for ken, r, rsc, and others. (Of course, Peter Weinberger's was his own [[http://spinroot.com/pico/pjw.html][iconic face]].) - -.image gopher/avatars.png - -Another Bell Labs activity led to Renee creating [[http://plan9.bell-labs.com/plan9/glenda.html][Glenda]], the Plan 9 mascot, a distant cousin of the WFMU gopher. - -.image gopher/glenda.png - -When we started the Go project we needed a logo, and Renee volunteered to draw it. It was featured on the first Go T-shirt and the Google Code site. - -.image gopher/logo.png - -For the open source launch in 2009, Renee suggested adapting the WFMU gopher as a mascot. And the Go gopher was born: - -.image gopher/gopher.png - -(The gopher has no name, and is called just the "Go gopher".) - -For the [[https://www.youtube.com/watch?v=-i0hat7pdpk#t=24m40s][launch]] of the [[https://developers.google.com/appengine/docs/go][Go App Engine runtime]] at Google I/O 2011 we engaged [[http://squishable.com][Squishable]] to manufacture the plush gophers. This was the first time the gopher was colored blue and appeared in three dimensions. The first prototype was kinda hairy: - -.image gopher/prototype.jpg - -But the second one was just right: - -.image gopher/plush.jpg - -Around the same time, Renee roughed out a gopher in clay. This inspired a refined sculpture that became a vinyl figurine made by [[http://www.kidrobot.com/][Kidrobot]]. The vinyls were first distributed at OSCON 2011. - -.image gopher/vinyl.jpg - -The gopher therefore exists in many forms, but has always been Renee's creation. It stands for the Go project and Go programmers everywhere, and is one of the most popular things in the Go world. - -The Go gopher is a character; a unique creation. Not any old gopher, just as Snoopy is not any old cartoon dog. - -The [[http://golang.org/doc/gopher/][gopher images]] are Creative Commons Attributions 3.0 licensed. That means you can play with the images but you must give credit to their creator (Renee French) wherever they are used. - -Here are a few gopher adaptations that people have used as mascots for user group mascots and similar organizations. - -.image gopher/usergroups.png - -They're cute and we like them, but by the Creative Commons rules the groups should give Renee credit, perhaps as a mention on the user group web site. - -The vinyl and plush gophers are copyrighted designs; accept no substitutes! But how can you get one? Their natural habitat is near high concentrations of Go programmers, and their worldwide population is growing. They may be purchased from the [[https://www.googlestore.com/Google+Redesign/Brands/Gopher/Go+Gopher+Squishable.axd][Google Store]], although the supply can be irregular. (These elusive creatures have been spotted in all [[http://www.flickr.com/photos/pinkhamsters/sets/72157627910426064/][kinds]] [[https://plus.google.com/107966629974328136037/posts/TcFA9zHzQB8][of]] [[https://twitter.com/goroutine][places]].) - -Perhaps the best way to get a gopher is to catch one in the wild at a Go conference. There are two big chances this year: [[http://gophercon.com/][GopherCon]] (Denver, April 24-26) and [[http://dotgo.eu][dotGo]] (Paris, October 10). - -.image gopher/portrait.jpg - -_(Photo_by_ [[https://twitter.com/noahhlo/status/437395572081688576][_Noah_Lorang_]]_.)_ diff --git a/content/gopher/avatars.png b/content/gopher/avatars.png deleted file mode 100644 index 707782d5..00000000 Binary files a/content/gopher/avatars.png and /dev/null differ diff --git a/content/gopher/glenda.png b/content/gopher/glenda.png deleted file mode 100644 index 5d7fe665..00000000 Binary files a/content/gopher/glenda.png and /dev/null differ diff --git a/content/gopher/gopher.png b/content/gopher/gopher.png deleted file mode 100644 index cb5e2e19..00000000 Binary files a/content/gopher/gopher.png and /dev/null differ diff --git a/content/gopher/header.jpg b/content/gopher/header.jpg deleted file mode 100644 index bcf63e98..00000000 Binary files a/content/gopher/header.jpg and /dev/null differ diff --git a/content/gopher/logo.png b/content/gopher/logo.png deleted file mode 100644 index d690ce01..00000000 Binary files a/content/gopher/logo.png and /dev/null differ diff --git a/content/gopher/plush.jpg b/content/gopher/plush.jpg deleted file mode 100644 index e02217fc..00000000 Binary files a/content/gopher/plush.jpg and /dev/null differ diff --git a/content/gopher/portrait.jpg b/content/gopher/portrait.jpg deleted file mode 100644 index 30227744..00000000 Binary files a/content/gopher/portrait.jpg and /dev/null differ diff --git a/content/gopher/prototype.jpg b/content/gopher/prototype.jpg deleted file mode 100644 index 3e9d3158..00000000 Binary files a/content/gopher/prototype.jpg and /dev/null differ diff --git a/content/gopher/usergroups.png b/content/gopher/usergroups.png deleted file mode 100644 index 642bc094..00000000 Binary files a/content/gopher/usergroups.png and /dev/null differ diff --git a/content/gopher/vinyl.jpg b/content/gopher/vinyl.jpg deleted file mode 100644 index ea8788a9..00000000 Binary files a/content/gopher/vinyl.jpg and /dev/null differ diff --git a/content/gopher/wfmu.jpg b/content/gopher/wfmu.jpg deleted file mode 100644 index 9b5d9a14..00000000 Binary files a/content/gopher/wfmu.jpg and /dev/null differ diff --git a/content/gopherchina.article b/content/gopherchina.article deleted file mode 100644 index d1fc66e0..00000000 --- a/content/gopherchina.article +++ /dev/null @@ -1,154 +0,0 @@ -GopherChina Trip Report -1 Jul 2015 - -Robert Griesemer -gri@golang.org - -* Introduction - -We have known for some time that Go is more popular in China than in any other -country. -According to Google Trends, most [[https://www.google.com/trends/explore#q=golang][searches for the term “golang”]] come from The People’s Republic than anywhere else. -[[http://herman.asia/why-is-go-popular-in-china][Others]] have speculated on -the same observation, yet so far we have had -[[https://news.ycombinator.com/item?id=8872400][sparse concrete information]] -about the phenomenon. - -The first Go conference in China, [[http://gopherchina.org/][GopherChina]], -seemed like an excellent opportunity to explore the situation by putting some -Western Gopher feet on Chinese ground. An actual invitation made it real and I -decided to accept and give a presentation about gofmt’s impact on software -development. - -.image gopherchina/image04.jpg - -_Hello,_Shanghai!_ - -The conference took place over an April weekend in Shanghai, in the -[[https://www.google.com/maps/place/Puruan+Bldg,+Pudong,+Shanghai,+China][Puruan Building]] -of the Shanghai Pudong Software Park, easily reachable by subway within an hour -or less from Shanghai’s more central parts. -Modelled after [[http://www.gophercon.com][GopherCon]], the conference was -single-track, with all talks presented in a conference room that fit about 400 -attendees. -It was organized by volunteers, lead by [[https://github.com/astaxie][Asta Xie]], -and with robust sponsorship from major industry names. According to the -organizers, many more people were hoping to attend than could be accommodated -due to space constraints. - -.image gopherchina/image01.jpg - -_The_welcoming_committee_with_Asta_Xie_(2nd_from_left),_the_primary_organizer._ - -Each attendee received a bag filled with the obligatory GopherChina t-shirt, -various sponsor-related informational brochures, stickers, and the occasional -stuffed “something” (no fluffy Gophers, though). At least one 3rd party vendor -was advertising technical books, including several original (not translated -from English) Go books. - -.image gopherchina/image05.jpg - -_Go_books!_ - -On first impression, the average attendee seemed pretty young, which made for -an enthusiastic crowd, and the event appeared well run. - -With the exception of my talk, all presentations were given in Mandarin and -thus were incomprehensible to me. Asta Xie, the primary organizer, assisted -with a few simultaneous translations whispered into my ear, and the occasional -English slide provided additional clues: “69GB” stands out even without any -Mandarin knowledge (more on that below). Consequently, I ended up listening to -a handful of presentations only, and instead spent much of my time talking with -attendees outside the main conference room. Yet judging from the slides, the -quality of most presentations seemed high, comparable with our experience at -GopherCon in Denver last year. Each talk got a one hour time slot which allowed -for plenty of technical detail, and many (dozens) of questions from an -enthusiastic audience. - -As expected, many of the presentations were about web services, backends for -mobile applications, and so on. Some of the systems appear to be huge by any -measure. -For instance, a talk by [[http://gopherchina.org/user/zhouyang][Yang Zhou]] -described a large-scale internal messaging system, used by -[[http://www.360.cn/][Qihoo 360]], a major Chinese software firm, all written -in Go. The presentation discussed how his team managed to reduce an original -heap size of 69GB (!) and the resulting long GC pauses of 3-6s to more -manageable numbers, and how they run millions of goroutines per machine, on a -fleet of thousands of machines. A future guest blog post is planned describing -this system in more detail. - -.image gopherchina/image03.jpg - -_Packed_conference_room_on_Saturday._ - -In another presentation, [[http://gopherchina.org/user/guofeng][Feng Guo]] from -[[https://www.daocloud.io/][DaoCloud]] talked about how they use Go in their -company for what they call the “continuous delivery” of applications. DaoCloud -takes care of automatically moving software hosted on GitHub (and Chinese -equivalents) to the cloud. A software developer simply pushes a new version on -GitHub and DaoCloud takes care of the rest: running tests, -[[https://www.docker.com/][Dockerizing]] it, and shipping it using your -preferred cloud service provider. - -Several speakers were from well-recognized major software firms (I showed the -conference program to non-technical people and they easily recognized several -of the firm’s names). Much more so than in the US, it seems Go is not just -hugely popular with newcomers and startups, but has very much found its way -into larger organizations and is employed at a scale that we are only starting -to see elsewhere. - -Not being an expert in web services myself, in my presentation I veered off the -general conference theme a bit by talking about -[[https://golang.org/cmd/gofmt/][gofmt]] and how its widespread use has started -to shape expectations not just for Go but other languages as well. -I presented in English but had my slides translated to Mandarin beforehand. Due -to the significant language barrier I wasn’t expecting too many questions on my -talk itself. -Instead I decided the keep it short and leave plenty of time for general -questions on Go, which the audience appreciated. - -.image gopherchina/image06.jpg - -_No_social_event_in_China_is_complete_without_fantastic_food._ - -A couple of days after the conference I visited the 4-year-old startup company -[[http://www.qiniu.com/][Qiniu]] (“Seven Bulls”), at the invitation of its -[[http://gopherchina.org/user/xushiwei][CEO]] Wei Hsu, facilitated and -translated with the help of Asta Xie. Qiniu is a cloud-based storage provider -for mobile applications; Wei Hsu presented at the conference and also happens -to be the author of one of the first Chinese books on Go (the leftmost one in -the picture above). - -.image gopherchina/image02.jpg -.image gopherchina/image00.jpg - -_Qiniu_lobby,_engineering._ - -Qiniu is an extremely successful all-Go shop, with about 160 employees, serving -over 150,000 companies and developers, storing over 50 Billion files, and -growing by over 500 Million files per day. When asked about the reasons for -Go’s success in China, Wei Hsu is quick to answer: PHP is extremely popular in -China, but relatively slow and not well-suited for large systems. Like in the -US, universities teach C++ and Java as primary languages, but for many -applications C++ is too complex a tool and Java too bulky. In his opinion, Go -now plays the role that traditionally belonged to PHP, but Go runs much faster, -is type safe, and scales more easily. He loves the fact that Go is simple and -applications are easy to deploy. He thought the language to be “perfect” for -them and his primary request was for a recommended or even standardized package -to easily access database systems. He did mention that they had GC problems in -the past but were able to work around them. Hopefully our upcoming 1.5 release -will address this. For Qiniu, Go appeared just at the right time and the right -(open source) place. - -According to Asta Xie, Qiniu is just one of many Go shops in the PRC. Large -companies such as Alibaba, Baidu, Tencent, and Weibo, are now all using Go in -one form or another. He pointed out that while Shanghai and neighboring cities -like [[https://www.google.com/maps/place/Suzhou,+Jiangsu,+China][Suzhou]] are -high-tech centres, even more software developers are found in the Beijing area. -For 2016,  Asta hopes to organize a larger (1000, perhaps 1500 people) -successor conference in Beijing. - -It appears that we have found the Go users in China: They are everywhere! - -_Some_of_the_GopherChina_materials,_including_videos,_are_now_available_alongside_Go_coursework_on_a_ [[http://www.imooc.com/view/407][_3rd_party_site_]]. - diff --git a/content/gopherchina/image00.jpg b/content/gopherchina/image00.jpg deleted file mode 100644 index 991badb5..00000000 Binary files a/content/gopherchina/image00.jpg and /dev/null differ diff --git a/content/gopherchina/image01.jpg b/content/gopherchina/image01.jpg deleted file mode 100644 index b42545e3..00000000 Binary files a/content/gopherchina/image01.jpg and /dev/null differ diff --git a/content/gopherchina/image02.jpg b/content/gopherchina/image02.jpg deleted file mode 100644 index 4ee3446a..00000000 Binary files a/content/gopherchina/image02.jpg and /dev/null differ diff --git a/content/gopherchina/image03.jpg b/content/gopherchina/image03.jpg deleted file mode 100644 index 4b8af5db..00000000 Binary files a/content/gopherchina/image03.jpg and /dev/null differ diff --git a/content/gopherchina/image04.jpg b/content/gopherchina/image04.jpg deleted file mode 100644 index 42ca71db..00000000 Binary files a/content/gopherchina/image04.jpg and /dev/null differ diff --git a/content/gopherchina/image05.jpg b/content/gopherchina/image05.jpg deleted file mode 100644 index b3529a74..00000000 Binary files a/content/gopherchina/image05.jpg and /dev/null differ diff --git a/content/gopherchina/image06.jpg b/content/gopherchina/image06.jpg deleted file mode 100644 index f4ec4708..00000000 Binary files a/content/gopherchina/image06.jpg and /dev/null differ diff --git a/content/gophercon.article b/content/gophercon.article deleted file mode 100644 index ee1911f8..00000000 --- a/content/gophercon.article +++ /dev/null @@ -1,51 +0,0 @@ -GopherCon 2014 Wrap Up -28 May 2014 - -Andrew Gerrand -adg@golang.org - -* Introduction - -In April this year 700 gophers descended upon Denver to attend [[http://www.gophercon.com/][GopherCon]], -the world's first large-scale Go conference, organized entirely by the community. -The three day event featured 24 talks and one panel discussion in a single track over two days, -followed by an informal "hack day" full of code, conversation, -and more than 4 hours (!) of lightning talks. - -.image gophercon/image02.jpg - -A [[http://confreaks.com/events/gophercon2014][complete set of video recordings]] is now available -(the [[https://github.com/gophercon/2014-talks][slides are here]]). - -Two keynotes framed the conference: - -- Rob Pike's opening talk [[https://www.youtube.com/watch?v=VoS7DsT1rdM]["Hello, Gophers!"]] ([[http://talks.golang.org/2014/hellogophers.slide][slides]]) discusses the history of Go by walking through two of the first Go programs. -- Andrew Gerrand's closing talk [[https://www.youtube.com/watch?v=dKGmK_Z1Zl0]["Go for gophers"]] ([[http://talks.golang.org/2014/go4gophers.slide][slides]]) explains the Go design philosophy through the lens of his personal experience learning the language. - -One talk that resonated with members of the Go team was Peter Bourgon's -[[https://www.youtube.com/watch?v=Y1-RLAl7iOI]["Best practices for Production Environments"]] -([[http://peter.bourgon.org/go-in-production/][notes]]). -From deployment to dependency management, it answers many frequently asked questions about Go use in the real world and is a must-see for anyone serious about building systems in Go. - -But, really, you should just [[http://confreaks.com/events/gophercon2014][watch them all]]. -They're great. - -The Go Gopher was everywhere. -Each attendee received one of the new pink and purple gophers, -which now accompany the original blue one: - -.image gophercon/image01.jpg - -The gopher was also seen wearing a cape on the side of the incredible [[https://coreos.com/][CoreOS]] bus: - -.image gophercon/image00.jpg - -Most of the Go team were at the conference, -and we were moved by the passion and dedication of the Go community. -It was a thrill to see the many different ways people are using the language. -It was also great to put faces to many of the names we know well through their contributions to the project. - -We would like to extend our thanks and congratulations to the GopherCon organizers (Brian Ketelsen and Erik St. Martin), the excellent speakers, and the tireless volunteers that pitched in to make the event such a success. -We look forward to GopherCon 2015, which promises to be bigger and better still. - -But if you really can't wait, we'll see you at [[http://www.dotgo.eu/][dotGo]] in Paris on the 10th of October! diff --git a/content/gophercon/image00.jpg b/content/gophercon/image00.jpg deleted file mode 100644 index b8dba8c0..00000000 Binary files a/content/gophercon/image00.jpg and /dev/null differ diff --git a/content/gophercon/image01.jpg b/content/gophercon/image01.jpg deleted file mode 100644 index dcd44579..00000000 Binary files a/content/gophercon/image01.jpg and /dev/null differ diff --git a/content/gophercon/image02.jpg b/content/gophercon/image02.jpg deleted file mode 100644 index 61003e01..00000000 Binary files a/content/gophercon/image02.jpg and /dev/null differ diff --git a/content/gophercon2015.article b/content/gophercon2015.article deleted file mode 100644 index b5cf6765..00000000 --- a/content/gophercon2015.article +++ /dev/null @@ -1,55 +0,0 @@ -GopherCon 2015 Roundup -28 Jul 2015 -Tags: conference, report, gopher - -Andrew Gerrand - -* Introduction - -A few weeks ago, Go programmers from around the world descended on Denver, -Colorado for GopherCon 2015. The two-day, single-track conference attracted -more than 1,250 attendees—nearly double last year's number—and featured 22 -talks presented by Go community members. - -.image gophercon2015.jpg _ 550 - -.html gophercon2015.caption - -Today the organizers have posted the videos online so you can now enjoy the -conference from afar: - -[[http://gophercon.com/schedule/8july/][Day 1]]: - -- Go, Open Source, Community — Russ Cox ([[https://www.youtube.com/watch?v=XvZOdpd_9tc][video]]) ([[http://blog.golang.org/open-source][text]]) -- Go kit: A Standard Library for Distributed Programming — Peter Bourgon ([[https://www.youtube.com/watch?v=1AjaZi4QuGo][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Go%20kit/go-kit.pdf][slides]]) -- Delve Into Go — Derek Parker ([[https://www.youtube.com/watch?v=InG72scKPd4][video]]) ([[http://go-talks.appspot.com/github.com/derekparker/talks/gophercon-2015/delve-into-go.slide][slides]]) -- How a complete beginner learned Go as her first backend language in 5 weeks — Audrey Lim ([[https://www.youtube.com/watch?v=fZh8uCInEfw][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Audrey%20Lim%20-%20How%20a%20Complete%20Beginner%20Picked%20Up%20Go%20as%20Her%20First%20Backend%20Language%20in%205%20weeks/audreylim_slides.pdf][slides]]) -- A Practical Guide to Preventing Deadlocks and Leaks in Go — Richard Fliam ([[https://www.youtube.com/watch?v=3EW1hZ8DVyw][video]]) -- Go GC: Solving the Latency Problem — Rick Hudson ([[https://www.youtube.com/watch?v=aiv1JOfMjm0][video]]) ([[http://talks.golang.org/2015/go-gc.pdf][slides]]) -- Simplicity and Go — Katherine Cox-Buday ([[https://www.youtube.com/watch?v=S6mEo_FHZ5Y][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Katherine%20Cox-Buday:%20Simplicity%20%26%20Go/Simplicity%20%26%20Go.pdf][slides]]) -- Rebuilding Parse.com in Go - an opinionated rewrite — Abhishek Kona ([[https://www.youtube.com/watch?v=_f9LS-OWfeA][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Abhishek%20Kona%20Rewriting%20Parse%20in%20GO/myslides.pdf][slides]]) -- Prometheus: Designing and Implementing a Modern Monitoring Solution in Go — Björn Rabenstein ([[https://www.youtube.com/watch?v=1V7eJ0jN8-E][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Bj%C3%B6rn%20Rabenstein%20-%20Prometheus/slides.pdf][slides]]) -- What Could Go Wrong? — Kevin Cantwell ([[https://www.youtube.com/watch?v=VC3QXZ-x5yI][video]]) -- The Roots of Go — Baishampayan Ghose ([[https://www.youtube.com/watch?v=0hPOopcJ8-E][video]]) ([[https://speakerdeck.com/bg/the-roots-of-go][slides]]) - -[[http://gophercon.com/schedule/9july/][Day 2]]: - -- The Evolution of Go — Robert Griesemer ([[https://www.youtube.com/watch?v=0ReKdcpNyQg][video]]) ([[http://talks.golang.org/2015/gophercon-goevolution.slide][slides]]) -- Static Code Analysis Using SSA — Ben Johnson ([[https://www.youtube.com/watch?v=D2-gaMvWfQY][video]]) ([[https://speakerdeck.com/benbjohnson/static-code-analysis-using-ssa][slides]]) -- Go on Mobile — Hana Kim ([[https://www.youtube.com/watch?v=sQ6-HyPxHKg][video]]) ([[http://talks.golang.org/2015/gophercon-go-on-mobile.slide][slides]]) -- Go Dynamic Tools — Dmitry Vyukov ([[https://www.youtube.com/watch?v=a9xrxRsIbSU][video]]) ([[http://talks.golang.org/2015/dynamic-tools.slide][slides]]) -- Embrace the Interface — Tomás Senart ([[https://www.youtube.com/watch?v=xyDkyFjzFVc][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Tom%C3%A1s%20Senart%20-%20Embrace%20the%20Interface/ETI.pdf][slides]]) -- Uptime: Building Resilient Services with Go — Blake Caldwell ([[https://www.youtube.com/watch?v=PyBJQA4clfc][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Blake%20Caldwell%20-%20Uptime:%20Building%20Resilient%20Services%20with%20Go/2015-GopherCon-Talk-Uptime.pdf][slides]]) -- Cayley: Building a Graph Database — Barak Michener ([[https://www.youtube.com/watch?v=-9kWbPmSyCI][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Barak%20Michener%20-%20Cayley:%20Building%20a%20Graph%20Database/Cayley%20-%20Building%20a%20Graph%20Database.pdf][slides]]) -- Code Generation For The Sake Of Consistency — Sarah Adams ([[https://www.youtube.com/watch?v=kGAgHwfjg1s][video]]) -- The Many Faces of Struct Tags — Sam Helman and Kyle Erf ([[https://www.youtube.com/watch?v=_SCRvMunkdA][video]]) ([[https://github.com/gophercon/2015-talks/blob/master/Sam%20Helman%20%26%20Kyle%20Erf%20-%20The%20Many%20Faces%20of%20Struct%20Tags/StructTags.pdf][slides]]) -- Betting the Company on Go and Winning — Kelsey Hightower ([[https://www.youtube.com/watch?v=wqVbLlHqAeY][video]]) -- How Go Was Made — Andrew Gerrand ([[https://www.youtube.com/watch?v=0ht89TxZZnk][video]]) ([[http://talks.golang.org/2015/how-go-was-made.slide][slides]]) - -The [[http://gophercon.com/schedule/10july/][hack day]] was also a ton of fun, -with hours of [[https://www.youtube.com/playlist?list=PL2ntRZ1ySWBeHqlHM8DmvS8axgbrpvF9b][lightning talks]] and a range of activities from programming robots -to a Magic: the Gathering tournament. - -Huge thanks to the event organizers Brian Ketelsen and Eric St. Martin and -their production team, the sponsors, the speakers, and the attendees for making -this such a fun and action-packed conference. Hope to see you there next year! diff --git a/content/gophercon2015.caption b/content/gophercon2015.caption deleted file mode 100644 index 3fb11d80..00000000 --- a/content/gophercon2015.caption +++ /dev/null @@ -1,5 +0,0 @@ -

-The Cowboy Gopher (a toy given to each attendee) watches over the ranch.
-Photograph by Nathan Youngman. Gopher by Renee French. - -

diff --git a/content/gophercon2015.jpg b/content/gophercon2015.jpg deleted file mode 100644 index 62ce0d6f..00000000 Binary files a/content/gophercon2015.jpg and /dev/null differ diff --git a/content/gophergala.article b/content/gophergala.article deleted file mode 100644 index af8d477e..00000000 --- a/content/gophergala.article +++ /dev/null @@ -1,35 +0,0 @@ -The Gopher Gala is the first worldwide Go hackathon -7 Jan 2015 - -Francesc Campoy -campoy@golang.org - -* Introduction - -The [[http://gophergala.com/][Gopher Gala]] is the first Go hackathon at a -global scale and will take place from January 23rd through the 25th. The event -is organized by the community, supported by the Go team, and sponsored by -Google among others. - -.image gophergala/fancygopher.jpg _ 300 -.caption _Fancy_Gopher,_by_ [[http://www.reneefrench.com/][_Renée_French_]] - -You can read about the rules of the hackathon -[[http://gophergala.com/rules/][here]], but if you know about -[[http://railsrumble.com/][Rails Rumble]] or -[[http://nodeknockout.com/][Node Knockout]] you already have a pretty good idea -of what to expect. - -During this event gophers from all around the globe will form teams to build -great applications using Go as the main tool. Afterwards, all the participants -will vote for their favorite applications and the 20 highest voted applications -will be ranked by a jury of renowned names from the community, including some -members of the Go core team. More information on the judging phase can be found -[[http://gophergala.com/judging/][here]]. - -And in case you needed one more reason to -[[https://gophers.typeform.com/to/wondM5][get involved]] there will be prizes! - -"What prizes?" you ask. Well, that’s a -[[http://gophergala.com/prizes/][secret]] until January 10th but we’re pretty -sure you won’t be disappointed. diff --git a/content/gophergala/fancygopher.jpg b/content/gophergala/fancygopher.jpg deleted file mode 100755 index ae0a3616..00000000 Binary files a/content/gophergala/fancygopher.jpg and /dev/null differ diff --git a/content/gos-declaration-syntax.article b/content/gos-declaration-syntax.article deleted file mode 100644 index fb7136f0..00000000 --- a/content/gos-declaration-syntax.article +++ /dev/null @@ -1,159 +0,0 @@ -Go's Declaration Syntax -7 Jul 2010 -Tags: c, syntax, ethos - -Rob Pike - -* Introduction - -Newcomers to Go wonder why the declaration syntax is different from the tradition established in the C family. In this post we'll compare the two approaches and explain why Go's declarations look as they do. - -* C syntax - -First, let's talk about C syntax. C took an unusual and clever approach to declaration syntax. Instead of describing the types with special syntax, one writes an expression involving the item being declared, and states what type that expression will have. Thus - - int x; - -declares x to be an int: the expression 'x' will have type int. In general, to figure out how to write the type of a new variable, write an expression involving that variable that evaluates to a basic type, then put the basic type on the left and the expression on the right. - -Thus, the declarations - - int *p; - int a[3]; - -state that p is a pointer to int because '*p' has type int, and that a is an array of ints because a[3] (ignoring the particular index value, which is punned to be the size of the array) has type int. - -What about functions? Originally, C's function declarations wrote the types of the arguments outside the parens, like this: - - int main(argc, argv) - int argc; - char *argv[]; - { /* ... */ } - -Again, we see that main is a function because the expression main(argc, argv) returns an int. In modern notation we'd write - - int main(int argc, char *argv[]) { /* ... */ } - -but the basic structure is the same. - -This is a clever syntactic idea that works well for simple types but can get confusing fast. The famous example is declaring a function pointer. Follow the rules and you get this: - - int (*fp)(int a, int b); - -Here, fp is a pointer to a function because if you write the expression (*fp)(a, b) you'll call a function that returns int. What if one of fp's arguments is itself a function? - - int (*fp)(int (*ff)(int x, int y), int b) - -That's starting to get hard to read. - -Of course, we can leave out the name of the parameters when we declare a function, so main can be declared - - int main(int, char *[]) - -Recall that argv is declared like this, - - char *argv[] - -so you drop the name from the middle of its declaration to construct its type. It's not obvious, though, that you declare something of type char *[] by putting its name in the middle. - -And look what happens to fp's declaration if you don't name the parameters: - - int (*fp)(int (*)(int, int), int) - -Not only is it not obvious where to put the name inside - - int (*)(int, int) - -it's not exactly clear that it's a function pointer declaration at all. And what if the return type is a function pointer? - - int (*(*fp)(int (*)(int, int), int))(int, int) - -It's hard even to see that this declaration is about fp. - -You can construct more elaborate examples but these should illustrate some of the difficulties that C's declaration syntax can introduce. - -There's one more point that needs to be made, though. Because type and declaration syntax are the same, it can be difficult to parse expressions with types in the middle. This is why, for instance, C casts always parenthesize the type, as in - - (int)M_PI - -* Go syntax - -Languages outside the C family usually use a distinct type syntax in declarations. Although it's a separate point, the name usually comes first, often followed by a colon. Thus our examples above become something like (in a fictional but illustrative language) - - x: int - p: pointer to int - a: array[3] of int - -These declarations are clear, if verbose - you just read them left to right. Go takes its cue from here, but in the interests of brevity it drops the colon and removes some of the keywords: - - x int - p *int - a [3]int - -There is no direct correspondence between the look of [3]int and how to use a in an expression. (We'll come back to pointers in the next section.) You gain clarity at the cost of a separate syntax. - -Now consider functions. Let's transcribe the declaration for main as it would read in Go, although the real main function in Go takes no arguments: - - func main(argc int, argv []string) int - -Superficially that's not much different from C, other than the change from `char` arrays to strings, but it reads well from left to right: - -function main takes an int and a slice of strings and returns an int. - -Drop the parameter names and it's just as clear - they're always first so there's no confusion. - - func main(int, []string) int - -One merit of this left-to-right style is how well it works as the types become more complex. Here's a declaration of a function variable (analogous to a function pointer in C): - - f func(func(int,int) int, int) int - -Or if f returns a function: - - f func(func(int,int) int, int) func(int, int) int - -It still reads clearly, from left to right, and it's always obvious which name is being declared - the name comes first. - -The distinction between type and expression syntax makes it easy to write and invoke closures in Go: - - sum := func(a, b int) int { return a+b } (3, 4) - -* Pointers - -Pointers are the exception that proves the rule. Notice that in arrays and slices, for instance, Go's type syntax puts the brackets on the left of the type but the expression syntax puts them on the right of the expression: - - var a []int - x = a[1] - -For familiarity, Go's pointers use the * notation from C, but we could not bring ourselves to make a similar reversal for pointer types. Thus pointers work like this - - var p *int - x = *p - -We couldn't say - - var p *int - x = p* - -because that postfix * would conflate with multiplication. We could have used the Pascal ^, for example: - - var p ^int - x = p^ - -and perhaps we should have (and chosen another operator for xor), because the prefix asterisk on both types and expressions complicates things in a number of ways. For instance, although one can write - - []int("hi") - -as a conversion, one must parenthesize the type if it starts with a *: - - (*int)(nil) - -Had we been willing to give up * as pointer syntax, those parentheses would be unnecessary. - -So Go's pointer syntax is tied to the familiar C form, but those ties mean that we cannot break completely from using parentheses to disambiguate types and expressions in the grammar. - -Overall, though, we believe Go's type syntax is easier to understand than C's, especially when things get complicated. - -* Notes - -Go's declarations read left to right. It's been pointed out that C's read in a spiral! See [[http://c-faq.com/decl/spiral.anderson.html][ The "Clockwise/Spiral Rule"]] by David Anderson. diff --git a/content/gothamgo.article b/content/gothamgo.article deleted file mode 100644 index 79a78b8d..00000000 --- a/content/gothamgo.article +++ /dev/null @@ -1,35 +0,0 @@ -GothamGo: gophers in the big apple -9 Jan 2015 - -Francesc Campoy -campoy@golang.org - -* Introduction - -Last November more than two hundred gophers from all across the United States got together for the first full-day Go conference in New York City. - -The diverse speaker lineup included university students, industry experts, and Go team members. - -And good news, everybody! All the talks were recorded and are available: - -.image gothamgo/gothamgo.jpg _ 600 - -- [[http://vimeo.com/115728346][Launching into Go]] _by_Kathy_Spardlin_ - a CockroachDB contributor provides pointers for people getting started with Go. -- [[http://vimeo.com/115782573][Error Handling]] _by_ [[https://twitter.com/goinggodotnet][_Bill_Kennedy_]] - ideas on how to use the Go error interface. -- [[http://vimeo.com/115776445][7 common mistakes in Go and how to avoid them]] _by_ [[https://twitter.com/spf13][_Steve_Francia_]] - the author of some popular Go libraries shares his experience. -- [[http://vimeo.com/115309491][Cancellation, Context, and Plumbing]] _by_ [[https://twitter.com/sajma][_Sameer_Ajmani_]] - the Google NYC Go team lead explains how cancellation works in Go, and how we're retrofitting the Google code base to use it. -- [[http://vimeo.com/115940605][Dethorning Package Management]] _by_ [[https://twitter.com/krarick][_Keith_Rarick_]] - the creator of godep talks about how to manage your dependencies well. -- [[http://vimeo.com/115940590][Everything You've Always Wanted to Know About Go Web Apps (But were afraid to ask)]] _by_ [[https://twitter.com/markbates][_Mark_Bates_]] - a survey of Go packages for building web apps, and insights into why there are so few Go "frameworks". -- [[http://vimeo.com/116215450][Building high-performance database applications using Aerospike]] _by_Chris_Stivers_ - a principal engineer at Aerospike shares his experiences building scalable applications. -- [[http://vimeo.com/115307069][The state of Go on the Android platform]] _by_ [[https://twitter.com/davidcrawshaw][_David_Crawshaw_]] - the leader of the Go on mobile platforms project tells us what’s coming up and shows a working demo. -- [[http://vimeo.com/116108566][Mutexes and Locks]] _by_ [[https://twitter.com/frazelledazzell][_Jessie_Frazelle_]] - a member of the Docker core team tells us about concurrency and what to do when things go wrong. -- [[http://vimeo.com/115618722][Gobot.io]] _by_ [[https://twitter.com/deadprogram][_Ron_Evans_]] - awesome robots controlled by Go, with demos! -- [[http://vimeo.com/114941260][Doing Go]] _by_ [[https://twitter.com/bryanl][_Bryan_Liles_]] - a DigitalOcean engineer delivers a hilarious comedy routine that happens to be about Go. -- [[http://vimeo.com/115308225][Things I learned teaching Go]] _by_ [[https://twitter.com/francesc][_Francesc_Campoy_]] - the Developer Advocate for the Go team shares his experience teaching Go and some advice on how to become a better gopher. - -Two more talks come from the Go meetup in New York City, which met the day before GothamGo: - -- [[http://vimeo.com/114975899][Benchmarking Go]] _by_ [[https://twitter.com/bbulkow][_Brian_Bulkowski_]] - the founder of Aerospike talks about profiling tools for Go and Linux, and micro benchmarks for goroutines, channels, buffers, and and other Go features. -- [[http://vimeo.com/114736889][Go Static Analysis Tools]] _by_Alan_Donovan_ - a member of the Go team at Google NY gives a guided tour of several static analysis tools designed to help Go programmers understand, navigate , and refactor their code. - -Make sure to have a look at all of those in preparation for the [[https://fosdem.org/][FOSDEM]] Go devroom FOSDEM in Brussels (Belgium) and [[http://www.gophercon.in/][gophercon.in]] in Bengaluru (India). diff --git a/content/gothamgo/gothamgo.jpg b/content/gothamgo/gothamgo.jpg deleted file mode 100644 index 5ed44759..00000000 Binary files a/content/gothamgo/gothamgo.jpg and /dev/null differ diff --git a/content/gouk15.article b/content/gouk15.article deleted file mode 100644 index 9b203987..00000000 --- a/content/gouk15.article +++ /dev/null @@ -1,43 +0,0 @@ -GolangUK 2015 -09 Oct 2015 -Tags: conference, golanguk - -Francesc Campoy - -* Introduction - -On August 21st the Go community gathered in London for the first edition of -[[https://golanguk.com][Golang UK]]. The conference featured two parallel -tracks and nearly 400 gophers attended. - -.image gouk15/gouk.jpg 300 _ - -The conference started with the opening keynote by [[https://twitter.com/calavera][David Calavera]] -called Crossing the Language Chasm ([[https://www.youtube.com/watch?v=JPVRnEZ4v_w&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -and continued with two concurrently executed [[http://golanguk.com/schedule/][tracks]]. - -Main track: - -- Stupid Gopher Tricks, by [[https://twitter.com/enneff][Andrew Gerrand]] ([[https://www.youtube.com/watch?v=UECh7X07m6E&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Complex Concurrency Patterns in Go, by [[https://twitter.com/eapache][Evan Huus]] ([[https://www.youtube.com/watch?v=2HOO5gIgyMg&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Code Analysis [no reading required], by [[https://twitter.com/francesc][Francesc Campoy]] ([[https://www.youtube.com/watch?v=oorX84tBMqo&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Go kit: a toolkit for microservices [[https://twitter.com/peterbourgon][Peter Bourgon]] ([[https://www.youtube.com/watch?v=aL6sd4d4hxk&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Dependency Management Conundrum, by [[https://twitter.com/goinggodotnet][William Kennedy]] ([[https://www.youtube.com/watch?v=CdhucJShJU8&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) - -Side track: - -- Building APIs, by [[https://twitter.com/matryer][Mat Ryer]] ([[https://www.youtube.com/watch?v=tIm8UkSf6RA&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Building a Bank with Go, by [[https://twitter.com/mattheath][Matt Heath]] ([[https://www.youtube.com/watch?v=cFJkLfujOts&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- CockroachDB: Make Data Easy, by [[https://twitter.com/bendarnell][Ben Darnell]] ([[https://www.youtube.com/watch?v=33oqpLmQ3LE&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Understanding memory allocation in Go, by [[https://twitter.com/DeanElbaz][Dean Elbaz]] ([[https://www.youtube.com/watch?v=zjoieOpy5hE&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) -- Whispered Secrets, by [[https://twitter.com/feyeleanor][Eleanor McHugh]] ([[https://www.youtube.com/watch?v=ViBRx-F4Z2U&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]) - -Finally [[https://twitter.com/dgryski][Damian Gryski]] took the stage for the -closing keynote ([[https://www.youtube.com/watch?v=IiSyFc10Jj0&list=PLDWZ5uzn69ezRJYeWxYNRMYebvf8DerHd][video]]), -giving an overview of how the Go community has evolved over time and hinting -to what the future might look like. - -On the day before the conference [[https://twitter.com/goinggodotnet][William Kennedy]] -gave a full day Go workshop. - -It was a great conference, so congratulations to the organizers and see you next year in London! diff --git a/content/gouk15/gouk.jpg b/content/gouk15/gouk.jpg deleted file mode 100644 index ba42756a..00000000 Binary files a/content/gouk15/gouk.jpg and /dev/null differ diff --git a/content/introducing-go-playground.article b/content/introducing-go-playground.article deleted file mode 100644 index cee14849..00000000 --- a/content/introducing-go-playground.article +++ /dev/null @@ -1,16 +0,0 @@ -Introducing the Go Playground -15 Sep 2010 -Tags: playground - -Andrew Gerrand - -* Introduction - -If you visit [[http://golang.org/][golang.org]] today you'll see our new look. We have given the site a new coat of paint and reorganized its content to make it easier to find. These changes are also reflected in the web interface of [[http://golang.org/cmd/godoc/][godoc]], the Go documentation tool. -But the real news is a prominent new feature: the [[http://golang.org/][Go Playground]]. -.image introducing-go-playground_Untitled.png - -The Playground allows anyone with a web browser to write Go code that we immediately compile, link, and run on our servers. There are a few example programs to get you started (see the "Examples" drop-down). We hope that this will give curious programmers an opportunity to try the language before [[http://golang.org/doc/install.html][installing it]], and experienced Go users a convenient place in which to experiment. Beyond the front page, this functionality has the potential to make our reference and tutorial materials more engaging. We hope to extend its use in the near future. - -Of course, there are some limitations to the kinds of programs you can run in the Playground. We can't simply accept arbitrary code and run it on our servers without restrictions. The programs build and run in a sandbox with a reduced standard library; the only communication your program has to the outside world is via standard output, and there are limits to CPU and memory use. As such, consider this just a taste of the wonderful world of Go; to have the full experience you'll need to [[http://golang.org/doc/install.html][download it yourself]]. -If you've been meaning to try Go but never got around to it, why not visit [[http://golang.org/][golang.org]] to try it right now? diff --git a/content/introducing-go-playground_Untitled.png b/content/introducing-go-playground_Untitled.png deleted file mode 100644 index 1694893d..00000000 Binary files a/content/introducing-go-playground_Untitled.png and /dev/null differ diff --git a/content/introducing-gofix.article b/content/introducing-gofix.article deleted file mode 100644 index 2e64e4ed..00000000 --- a/content/introducing-gofix.article +++ /dev/null @@ -1,55 +0,0 @@ -Introducing Gofix -15 Apr 2011 -Tags: gofix, technical - -Russ Cox - -* Introduction - -The next Go release will include significant API changes in several fundamental Go packages. Code that [[http://codereview.appspot.com/4239076][implements an HTTP server handler]], [[http://codereview.appspot.com/4244055][calls `net.Dial`]], [[http://codereview.appspot.com/4357052][calls `os.Open`]], or [[http://codereview.appspot.com/4281055][uses the reflect package]] will not build unless it is updated to use the new APIs. Now that our releases are [[http://blog.golang.org/2011/03/go-becomes-more-stable.html][more stable and less frequent]], this will be a common situation. Each of these API changes happened in a different weekly snapshot and might have been manageable on its own; together, however, they represent a significant amount of manual effort to update existing code. - -[[http://golang.org/cmd/fix/][Gofix]] is a new tool that reduces the amount of effort it takes to update existing code. It reads a program from a source file, looks for uses of old APIs, rewrites them to use the current API, and writes the program back to the file. Not all API changes preserve all the functionality of an old API, so gofix cannot always do a perfect job. When gofix cannot rewrite a use of an old API, it prints a warning giving the file name and line number of the use, so that a developer can examine and rewrite the code. Gofix takes care of the easy, repetitive, tedious changes, so that a developer can focus on the ones that truly merit attention. - -Each time we make a significant API change we’ll add code to gofix to take care of the conversion, as much as mechanically possible. When you update to a new Go release and your code no longer builds, just run gofix on your source directory. - -You can extend gofix to support changes to your own APIs. The gofix program is a simple driver around plugins called fixes that each handle a particular API change. Right now, writing a new fix requires doing some scanning and rewriting of the go/ast syntax tree, usually in proportion to how complex the API changes are. If you want to explore, the [[https://code.google.com/p/go/source/browse/src/cmd/fix/netdial.go?name=go1][`netdialFix`]], [[http://code.google.com/p/go/source/browse/src/cmd/fix/osopen.go?name=go1][`osopenFix`]], [[http://code.google.com/p/go/source/browse/src/cmd/fix/httpserver.go?name=go1][`httpserverFix`]], and [[http://code.google.com/p/go/source/browse/src/cmd/fix/reflect.go?name=go1][`reflectFix`]] are all illustrative examples, in increasing order of complexity. - -We write Go code too, of course, and our code is just as affected by these API changes as yours. Typically, we write the gofix support at the same time as the API change and then use gofix to rewrite the uses in the main source tree. We use gofix to update other Go code bases and our personal projects. We even use gofix to update Google’s internal source tree when it is time to build against a new Go release. - -As an example, gofix can rewrite code like [[http://codereview.appspot.com/4353043/diff/10001/src/pkg/fmt/print.go#newcode657][this snippet from `fmt/print.go`]]: - - switch f := value.(type) { - case *reflect.BoolValue: - p.fmtBool(f.Get(), verb, field) - case *reflect.IntValue: - p.fmtInt64(f.Get(), verb, field) - // ... - case reflect.ArrayOrSliceValue: - // Byte slices are special. - if f.Type().(reflect.ArrayOrSliceType).Elem().Kind() == reflect.Uint8 { - // ... - } - // ... - } - -to adapt it to the new reflect API: - - switch f := value; f.Kind() { - case reflect.Bool: - p.fmtBool(f.Bool(), verb, field) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - p.fmtInt64(f.Int(), verb, field) - // ... - case reflect.Array, reflect.Slice: - // Byte slices are special. - if f.Type().Elem().Kind() == reflect.Uint8 { - // ... - } - // ... - } - -Nearly every line above changed in some small way. The changes involved in the rewrite are extensive but nearly entirely mechanical, just the kind of thing that computers are great at doing. - -Gofix is possible because Go has support in its standard libraries for [[http://golang.org/pkg/go/parser][parsing Go source files into syntax trees]] and also for [[http://golang.org/pkg/go/printer][printing those syntax trees back to Go source code]]. Importantly, the Go printing library prints a program in the official format (typically enforced via the gofmt tool), allowing gofix to make mechanical changes to Go programs without causing spurious formatting changes. In fact, one of the key motivations for creating gofmt—perhaps second only to avoiding debates about where a particular brace belongs—was to simplify the creation of tools that rewrite Go programs, as gofix does. - -Gofix has already made itself indispensable. In particular, the recent reflect changes would have been unpalatable without automated conversion, and the reflect API badly needed to be redone. Gofix gives us the ability to fix mistakes or completely rethink package APIs without worrying about the cost of converting existing code. We hope you find gofix as useful and convenient as we have. diff --git a/content/io2014.article b/content/io2014.article deleted file mode 100644 index d115acb9..00000000 --- a/content/io2014.article +++ /dev/null @@ -1,78 +0,0 @@ -Go at Google I/O and Gopher SummerFest -6 Oct 2014 -Tags: conference, io, report - -Francesc Campoy - -* Introduction - -The week of June 23rd was a good week for gophers in San Francisco. Go was a big -part of [[https://www.google.com/events/io][Google I/O]] on Wednesday and -Thursday, and on Monday we took advantage of the large gopher population to run -the [[http://www.meetup.com/golangsf/events/170421402/][Go SummerFest]], a -special instance of the [[http://www.meetup.com/golangsf][GoSF]] meetup. This -blog post is a recap of both events. - -* Gopher SummerFest - -.image io2014/summerfest.jpg _ 800 - -On the Monday, more than 200 gophers gathered at the Google office in San -Francisco to hear a series of talks: - -- The State of Go, ([[https://docs.google.com/a/golang.org/file/d/0B-nws9GU_6qVQm9tdncxWnJGUTQ/edit][slides]] and [[http://www.hakkalabs.co/articles/state-go][video]]) by Andrew Gerrand. -- I was wrong, again! ([[https://docs.google.com/a/golang.org/file/d/0B-nws9GU_6qVTTUwS25qaFlzMGs/edit][slides]] and [[http://www.hakkalabs.co/articles/why-we-use-go][video]]), by Derek Collison. -- Go at Splice ([[https://docs.google.com/a/golang.org/file/d/0B-nws9GU_6qVZklnNnJITlhSbXc/edit][slides]]), by Matt Aimonetti -- Quick testing with quick ([[https://docs.google.com/a/golang.org/file/d/0B-nws9GU_6qVVElmOHpwamp4aTA/edit][slides]]), by Evan Shaw -- Something about Go (no slides), by Blake Mizerany. - -More comments and pictures from the event are available on the -[[http://www.meetup.com/golangsf/events/170421402/][meetup event page]]. - -* Go at Google I/O - -On the Wednesday and Thursday, Go was at Google I/O in two different -formats: the Go booth in the sandbox area and the Go code labs available in the -code lab area and all around the world through -[[https://www.google.com/events/io/io-extended][I/O Extended]]. - -** The Go booth - -.image io2014/booth.jpg _ 800 - -The Go booth was part of the Developer Sandbox area. - -For the two days of the conference, some gophers from Google and other -companies gave a series of talks and demonstrations. The talks were not -recorded, but the slides and some screencasts and blog posts will be shared -soon. - -- Organizing Go Code, by David Crawshaw. ([[https://talks.golang.org/2014/organizeio.slide#1][slides]]) -- Testing Techniques, by Andrew Gerrand. ([[https://www.youtube.com/watch?v=ndmB0bj7eyw][video]] and [[https://talks.golang.org/2014/testing.slide#1][slides]]) -- Go for Java Developers, by Francesc Campoy. ([[https://talks.golang.org/2014/go4java.slide#1][slides]]) -- Camlistore: Android, ARM, App Engine, Everywhere, by Brad Fitzpatrick. ([[https://talks.golang.org/2014/camlistore.slide#1][slides]]) -- Go Compilation Complexities, by Ian Lance Taylor. ([[https://talks.golang.org/2014/compiling.slide#1][slides]]) -- SourceGraph: a Code Search Engine in Go, by Quinn Slack. ([[https://youtu.be/-DpKaoPz8l8][video]] and [[https://go-talks.appspot.com/github.com/sourcegraph/talks/google-io-2014/gio2014.slide#1][slides]]) - - -We also organized Q&A sessions and lightning talks by members of the Go -community: - -- [[https://twitter.com/bradrydzewski][Brad Rydzewski]] talked about his project [[https://drone.io/][drone.io]]. -- [[https://twitter.com/barakmich][Barak Michener]] presented [[https://github.com/google/cayley][Cayley]], an open source graph database. -- [[https://twitter.com/mattetti][Matt Aimonetti]] discussed how Go is used at [[https://splice.com/][Splice]]. -- [[https://twitter.com/ssougou][Sugu Sougoumarane]] talked about how [[https://github.com/youtube/vitess][vitess]] solved scalability problems at YouTube. ([[https://youtu.be/midJ6b1LkA0][video]]) - -** The Go code lab - -This year attendees of Google I/O had a code lab area with self-service -computers where they could sit and learn Go. The code labs were also available -to anyone through the Google I/O extended brand. You can try it yourself at -[[https://io2014codelabs.appspot.com/][io2014codelabs.appspot.com]]. - -.image io2014/collage.jpg _ 800 - -* Conclusion - -Thanks to the organizers, speakers, and attendees who helped make these events a -great success. See you next year. (Or at [[http://dotgo.eu][dotGo]] this week!) \ No newline at end of file diff --git a/content/io2014/booth.jpg b/content/io2014/booth.jpg deleted file mode 100644 index 50502b3a..00000000 Binary files a/content/io2014/booth.jpg and /dev/null differ diff --git a/content/io2014/collage.jpg b/content/io2014/collage.jpg deleted file mode 100644 index 577e7a3e..00000000 Binary files a/content/io2014/collage.jpg and /dev/null differ diff --git a/content/io2014/crowd.jpg b/content/io2014/crowd.jpg deleted file mode 100755 index f943f370..00000000 Binary files a/content/io2014/crowd.jpg and /dev/null differ diff --git a/content/io2014/summerfest.jpg b/content/io2014/summerfest.jpg deleted file mode 100755 index d5ee49c1..00000000 Binary files a/content/io2014/summerfest.jpg and /dev/null differ diff --git a/content/json-and-go.article b/content/json-and-go.article deleted file mode 100644 index 4a1ae369..00000000 --- a/content/json-and-go.article +++ /dev/null @@ -1,253 +0,0 @@ -JSON and Go -25 Jan 2011 -Tags: json, technical - -Andrew Gerrand - -* Introduction - -JSON (JavaScript Object Notation) is a simple data interchange format. Syntactically it resembles the objects and lists of JavaScript. It is most commonly used for communication between web back-ends and JavaScript programs running in the browser, but it is used in many other places, too. Its home page, [[http://json.org][json.org]], provides a wonderfully clear and concise definition of the standard. - -With the [[http://golang.org/pkg/encoding/json/][json package]] it's a snap to read and write JSON data from your Go programs. - -* Encoding - -To encode JSON data we use the [[http://golang.org/pkg/encoding/json/#Marshal][`Marshal`]] function. - - func Marshal(v interface{}) ([]byte, error) - -Given the Go data structure, `Message`, - - type Message struct { - Name string - Body string - Time int64 - } - -and an instance of `Message` - - m := Message{"Alice", "Hello", 1294706395881547000} - -we can marshal a JSON-encoded version of m using `json.Marshal`: - - b, err := json.Marshal(m) - -If all is well, `err` will be `nil` and `b` will be a `[]byte` containing this JSON data: - - b == []byte(`{"Name":"Alice","Body":"Hello","Time":1294706395881547000}`) - -Only data structures that can be represented as valid JSON will be encoded: - -- JSON objects only support strings as keys; to encode a Go map type it must be of the form `map[string]T` (where `T` is any Go type supported by the json package). - -- Channel, complex, and function types cannot be encoded. - -- Cyclic data structures are not supported; they will cause `Marshal` to go into an infinite loop. - -- Pointers will be encoded as the values they point to (or 'null' if the pointer is `nil`). - -The json package only accesses the exported fields of struct types (those that begin with an uppercase letter). Therefore only the the exported fields of a struct will be present in the JSON output. - -* Decoding - -To decode JSON data we use the [[http://golang.org/pkg/encoding/json/#Unmarshal][`Unmarshal`]] function. - - func Unmarshal(data []byte, v interface{}) error - -We must first create a place where the decoded data will be stored - - var m Message - -and call `json.Unmarshal`, passing it a `[]byte` of JSON data and a pointer to `m` - - err := json.Unmarshal(b, &m) - -If `b` contains valid JSON that fits in `m`, after the call `err` will be `nil` and the data from `b` will have been stored in the struct `m`, as if by an assignment like: - - m = Message{ - Name: "Alice", - Body: "Hello", - Time: 1294706395881547000, - } - -How does `Unmarshal` identify the fields in which to store the decoded data? For a given JSON key `"Foo"`, `Unmarshal` will look through the destination struct's fields to find (in order of preference): - -- An exported field with a tag of `"Foo"` (see the [[http://golang.org/ref/spec#Struct_types][Go spec]] for more on struct tags), - -- An exported field named `"Foo"`, or - -- An exported field named `"FOO"` or `"FoO"` or some other case-insensitive match of `"Foo"`. - -What happens when the structure of the JSON data doesn't exactly match the Go type? - - b := []byte(`{"Name":"Bob","Food":"Pickle"}`) - var m Message - err := json.Unmarshal(b, &m) - -`Unmarshal` will decode only the fields that it can find in the destination type. In this case, only the Name field of m will be populated, and the Food field will be ignored. This behavior is particularly useful when you wish to pick only a few specific fields out of a large JSON blob. It also means that any unexported fields in the destination struct will be unaffected by `Unmarshal`. - -But what if you don't know the structure of your JSON data beforehand? - -* Generic JSON with interface{} - -The `interface{}` (empty interface) type describes an interface with zero methods. Every Go type implements at least zero methods and therefore satisfies the empty interface. - -The empty interface serves as a general container type: - - var i interface{} - i = "a string" - i = 2011 - i = 2.777 - -A type assertion accesses the underlying concrete type: - - r := i.(float64) - fmt.Println("the circle's area", math.Pi*r*r) - -Or, if the underlying type is unknown, a type switch determines the type: - - switch v := i.(type) { - case int: - fmt.Println("twice i is", v*2) - case float64: - fmt.Println("the reciprocal of i is", 1/v) - case string: - h := len(v) / 2 - fmt.Println("i swapped by halves is", v[h:]+v[:h]) - default: - // i isn't one of the types above - } - -The json package uses `map[string]interface{}` and -`[]interface{}` values to store arbitrary JSON objects and arrays; -it will happily unmarshal any valid JSON blob into a plain -`interface{}` value. The default concrete Go types are: - -- `bool` for JSON booleans, - -- `float64` for JSON numbers, - -- `string` for JSON strings, and - -- `nil` for JSON null. - -* Decoding arbitrary data - -Consider this JSON data, stored in the variable `b`: - - b := []byte(`{"Name":"Wednesday","Age":6,"Parents":["Gomez","Morticia"]}`) - -Without knowing this data's structure, we can decode it into an `interface{}` value with `Unmarshal`: - - var f interface{} - err := json.Unmarshal(b, &f) - -At this point the Go value in `f` would be a map whose keys are strings and whose values are themselves stored as empty interface values: - - f = map[string]interface{}{ - "Name": "Wednesday", - "Age": 6, - "Parents": []interface{}{ - "Gomez", - "Morticia", - }, - } - -To access this data we can use a type assertion to access `f`'s underlying `map[string]interface{}`: - - m := f.(map[string]interface{}) - -We can then iterate through the map with a range statement and use a type switch to access its values as their concrete types: - - for k, v := range m { - switch vv := v.(type) { - case string: - fmt.Println(k, "is string", vv) - case int: - fmt.Println(k, "is int", vv) - case []interface{}: - fmt.Println(k, "is an array:") - for i, u := range vv { - fmt.Println(i, u) - } - default: - fmt.Println(k, "is of a type I don't know how to handle") - } - } - -In this way you can work with unknown JSON data while still enjoying the benefits of type safety. - -* Reference Types - -Let's define a Go type to contain the data from the previous example: - - type FamilyMember struct { - Name string - Age int - Parents []string - } - - var m FamilyMember - err := json.Unmarshal(b, &m) - -Unmarshaling that data into a `FamilyMember` value works as expected, but if we look closely we can see a remarkable thing has happened. With the var statement we allocated a `FamilyMember` struct, and then provided a pointer to that value to `Unmarshal`, but at that time the `Parents` field was a `nil` slice value. To populate the `Parents` field, `Unmarshal` allocated a new slice behind the scenes. This is typical of how `Unmarshal` works with the supported reference types (pointers, slices, and maps). - -Consider unmarshaling into this data structure: - - type Foo struct { - Bar *Bar - } - -If there were a `Bar` field in the JSON object, `Unmarshal` would allocate a new `Bar` and populate it. If not, `Bar` would be left as a `nil` pointer. - -From this a useful pattern arises: if you have an application that receives a few distinct message types, you might define "receiver" structure like - - type IncomingMessage struct { - Cmd *Command - Msg *Message - } - -and the sending party can populate the `Cmd` field and/or the `Msg` field of the top-level JSON object, depending on the type of message they want to communicate. `Unmarshal`, when decoding the JSON into an `IncomingMessage` struct, will only allocate the data structures present in the JSON data. To know which messages to process, the programmer need simply test that either `Cmd` or `Msg` is not `nil`. - -* Streaming Encoders and Decoders - -The json package provides `Decoder` and `Encoder` types to support the common operation of reading and writing streams of JSON data. The `NewDecoder` and `NewEncoder` functions wrap the [[http://golang.org/pkg/io/#Reader][`io.Reader`]] and [[http://golang.org/pkg/io/#Writer][`io.Writer`]] interface types. - - func NewDecoder(r io.Reader) *Decoder - func NewEncoder(w io.Writer) *Encoder - -Here's an example program that reads a series of JSON objects from standard input, removes all but the `Name` field from each object, and then writes the objects to standard output: - - package main - - import ( - "encoding/json" - "log" - "os" - ) - - func main() { - dec := json.NewDecoder(os.Stdin) - enc := json.NewEncoder(os.Stdout) - for { - var v map[string]interface{} - if err := dec.Decode(&v); err != nil { - log.Println(err) - return - } - for k := range v { - if k != "Name" { - delete(v, k) - } - } - if err := enc.Encode(&v); err != nil { - log.Println(err) - } - } - } - -Due to the ubiquity of Readers and Writers, these `Encoder` and `Decoder` types can be used in a broad range of scenarios, such as reading and writing to HTTP connections, WebSockets, or files. - -* References - -For more information see the [[http://golang.org/pkg/encoding/json/][json package documentation]]. For an example usage of json see the source files of the [[http://golang.org/pkg/net/rpc/jsonrpc/][jsonrpc package]]. diff --git a/content/json-rpc-tale-of-interfaces.article b/content/json-rpc-tale-of-interfaces.article deleted file mode 100644 index 286ad11a..00000000 --- a/content/json-rpc-tale-of-interfaces.article +++ /dev/null @@ -1,36 +0,0 @@ -JSON-RPC: a tale of interfaces -27 Apr 2010 -Tags: json, rpc, technical - -Andrew Gerrand - -* Introduction - -Here we present an example where Go's [[http://golang.org/doc/effective_go.html#interfaces_and_types][interfaces]] made it easy to refactor some existing code to make it more flexible and extensible. Originally, the standard library's [[http://golang.org/pkg/net/rpc/][RPC package]] used a custom wire format called [[http://golang.org/pkg/encoding/gob/][gob]]. For a particular application, we wanted to use [[http://golang.org/pkg/encoding/json/][JSON]] as an alternate wire format. - -We first defined a pair of interfaces to describe the functionality of the existing wire format, one for the client, and one for the server (depicted below). - - type ServerCodec interface { - ReadRequestHeader(*Request) error - ReadRequestBody(interface{}) error - WriteResponse(*Response, interface{}) error - Close() error - } - -On the server side, we then changed two internal function signatures to accept the `ServerCodec` interface instead of our existing `gob.Encoder`. Here's one of them: - - func sendResponse(sending *sync.Mutex, req *Request, - reply interface{}, enc *gob.Encoder, errmsg string) - -became - - func sendResponse(sending *sync.Mutex, req *Request, - reply interface{}, enc ServerCodec, errmsg string) - -We then wrote a trivial `gobServerCodec` wrapper to reproduce the original functionality. From there it is simple to build a `jsonServerCodec`. - -After some similar changes to the client side, this was the full extent of the work we needed to do on the RPC package. This whole exercise took about 20 minutes! After tidying up and testing the new code, the [[http://code.google.com/p/go/source/diff?spec=svn9daf796ebf1cae97b2fcf760a4ab682f1f063f29&r=9daf796ebf1cae97b2fcf760a4ab682f1f063f29&format=side&path=/src/pkg/rpc/server.go][final changeset]] was submitted. - -In an inheritance-oriented language like Java or C++, the obvious path would be to generalize the RPC class, and create JsonRPC and GobRPC subclasses. However, this approach becomes tricky if you want to make a further generalization orthogonal to that hierarchy. (For example, if you were to implement an alternate RPC standard). In our Go package, we took a route that is both conceptually simpler and requires less code be written or changed. - -A vital quality for any codebase is maintainability. As needs change, it is essential to adapt your code easily and cleanly, lest it become unwieldy to work with. We believe Go's lightweight, composition-oriented type system provides a means of structuring code that scales. diff --git a/content/laws-of-reflection.article b/content/laws-of-reflection.article deleted file mode 100644 index a95ac0eb..00000000 --- a/content/laws-of-reflection.article +++ /dev/null @@ -1,334 +0,0 @@ -The Laws of Reflection -6 Sep 2011 -Tags: interface, reflect, type, technical - -Rob Pike - -* Introduction - -Reflection in computing is the ability of a program to examine its own structure, particularly through types; it's a form of metaprogramming. It's also a great source of confusion. - -In this article we attempt to clarify things by explaining how reflection works in Go. Each language's reflection model is different (and many languages don't support it at all), but this article is about Go, so for the rest of this article the word "reflection" should be taken to mean "reflection in Go". - -* Types and interfaces - -Because reflection builds on the type system, let's start with a refresher about types in Go. - -Go is statically typed. Every variable has a static type, that is, exactly one type known and fixed at compile time: `int`, `float32`, `*MyType`, `[]byte`, and so on. If we declare - - type MyInt int - - var i int - var j MyInt - -then `i` has type `int` and `j` has type `MyInt`. The variables `i` and `j` have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion. - -One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface's methods. A well-known pair of examples is `io.Reader` and `io.Writer`, the types `Reader` and `Writer` from the [[http://golang.org/pkg/io/][io package]]: - - // Reader is the interface that wraps the basic Read method. - type Reader interface { - Read(p []byte) (n int, err error) - } - - // Writer is the interface that wraps the basic Write method. - type Writer interface { - Write(p []byte) (n int, err error) - } - -Any type that implements a `Read` (or `Write`) method with this signature is said to implement `io.Reader` (or `io.Writer`). For the purposes of this discussion, that means that a variable of type `io.Reader` can hold any value whose type has a `Read` method: - - var r io.Reader - r = os.Stdin - r = bufio.NewReader(r) - r = new(bytes.Buffer) - // and so on - -It's important to be clear that whatever concrete value `r` may hold, `r`'s type is always `io.Reader`: Go is statically typed and the static type of `r` is `io.Reader`. - -An extremely important example of an interface type is the empty interface: - - interface{} - -It represents the empty set of methods and is satisfied by any value at all, since any value has zero or more methods. - -Some people say that Go's interfaces are dynamically typed, but that is misleading. They are statically typed: a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface. - -We need to be precise about all this because reflection and interfaces are closely related. - -* The representation of an interface - -Russ Cox has written a [[http://research.swtch.com/2009/12/go-data-structures-interfaces.html][ detailed blog post]] about the representation of interface values in Go. It's not necessary to repeat the full story here, but a simplified summary is in order. - -A variable of interface type stores a pair: the concrete value assigned to the variable, and that value's type descriptor. To be more precise, the value is the underlying concrete data item that implements the interface and the type describes the full type of that item. For instance, after - - var r io.Reader - tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0) - if err != nil { - return nil, err - } - r = tty - -`r` contains, schematically, the (value, type) pair, (`tty`, `*os.File`). Notice that the type `*os.File` implements methods other than `Read`; even though the interface value provides access only to the `Read` method, the value inside carries all the type information about that value. That's why we can do things like this: - - var w io.Writer - w = r.(io.Writer) - -The expression in this assignment is a type assertion; what it asserts is that the item inside `r` also implements `io.Writer`, and so we can assign it to `w`. After the assignment, `w` will contain the pair (`tty`, `*os.File`). That's the same pair as was held in `r`. The static type of the interface determines what methods may be invoked with an interface variable, even though the concrete value inside may have a larger set of methods. - -Continuing, we can do this: - - var empty interface{} - empty = w - -and our empty interface value `empty` will again contain that same pair, (`tty`, `*os.File`). That's handy: an empty interface can hold any value and contains all the information we could ever need about that value. - -(We don't need a type assertion here because it's known statically that `w` satisfies the empty interface. In the example where we moved a value from a `Reader` to a `Writer`, we needed to be explicit and use a type assertion because `Writer`'s methods are not a subset of `Reader`'s.) - -One important detail is that the pair inside an interface always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values. - -Now we're ready to reflect. - -* The first law of reflection - -* 1. Reflection goes from interface value to reflection object. - -At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. To get started, there are two types we need to know about in [[http://golang.org/pkg/reflect/][package reflect]]: [[http://golang.org/pkg/reflect/#Type][Type]] and [[http://golang.org/pkg/reflect/#Value][Value]]. Those two types give access to the contents of an interface variable, and two simple functions, called `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value. (Also, from the `reflect.Value` it's easy to get to the `reflect.Type`, but let's keep the `Value` and `Type` concepts separate for now.) - -Let's start with `TypeOf`: - - package main - - import ( - "fmt" - "reflect" - ) - - func main() { - var x float64 = 3.4 - fmt.Println("type:", reflect.TypeOf(x)) - } - -This program prints - - type: float64 - -You might be wondering where the interface is here, since the program looks like it's passing the `float64` variable `x`, not an interface value, to `reflect.TypeOf`. But it's there; as [[http://golang.org/pkg/reflect/#TypeOf][godoc reports]], the signature of `reflect.TypeOf` includes an empty interface: - - // TypeOf returns the reflection Type of the value in the interface{}. - func TypeOf(i interface{}) Type - -When we call `reflect.TypeOf(x)`, `x` is first stored in an empty interface, which is then passed as the argument; `reflect.TypeOf` unpacks that empty interface to recover the type information. - -The `reflect.ValueOf` function, of course, recovers the value (from here on we'll elide the boilerplate and focus just on the executable code): - - var x float64 = 3.4 - fmt.Println("value:", reflect.ValueOf(x)) - -prints - - value: - -Both `reflect.Type` and `reflect.Value` have lots of methods to let us examine and manipulate them. One important example is that `Value` has a `Type` method that returns the `Type` of a `reflect.Value`. Another is that both `Type` and `Value` have a `Kind` method that returns a constant indicating what sort of item is stored: `Uint`, `Float64`, `Slice`, and so on. Also methods on `Value` with names like `Int` and `Float` let us grab values (as `int64` and `float64`) stored inside: - - var x float64 = 3.4 - v := reflect.ValueOf(x) - fmt.Println("type:", v.Type()) - fmt.Println("kind is float64:", v.Kind() == reflect.Float64) - fmt.Println("value:", v.Float()) - -prints - - type: float64 - kind is float64: true - value: 3.4 - -There are also methods like `SetInt` and `SetFloat` but to use them we need to understand settability, the subject of the third law of reflection, discussed below. - -The reflection library has a couple of properties worth singling out. First, to keep the API simple, the "getter" and "setter" methods of `Value` operate on the largest type that can hold the value: `int64` for all the signed integers, for instance. That is, the `Int` method of `Value` returns an `int64` and the `SetInt` value takes an `int64`; it may be necessary to convert to the actual type involved: - - var x uint8 = 'x' - v := reflect.ValueOf(x) - fmt.Println("type:", v.Type()) // uint8. - fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true. - x = uint8(v.Uint()) // v.Uint returns a uint64. - -The second property is that the `Kind` of a reflection object describes the underlying type, not the static type. If a reflection object contains a value of a user-defined integer type, as in - - type MyInt int - var x MyInt = 7 - v := reflect.ValueOf(x) - -the `Kind` of `v` is still `reflect.Int`, even though the static type of `x` is `MyInt`, not `int`. In other words, the `Kind` cannot discriminate an int from a `MyInt` even though the `Type` can. - -* The second law of reflection - -* 2. Reflection goes from reflection object to interface value. - -Like physical reflection, reflection in Go generates its own inverse. - -Given a `reflect.Value` we can recover an interface value using the `Interface` method; in effect the method packs the type and value information back into an interface representation and returns the result: - - // Interface returns v's value as an interface{}. - func (v Value) Interface() interface{} - -As a consequence we can say - - y := v.Interface().(float64) // y will have type float64. - fmt.Println(y) - -to print the `float64` value represented by the reflection object `v`. - -We can do even better, though. The arguments to `fmt.Println`, `fmt.Printf` and so on are all passed as empty interface values, which are then unpacked by the `fmt` package internally just as we have been doing in the previous examples. Therefore all it takes to print the contents of a `reflect.Value` correctly is to pass the result of the `Interface` method to the formatted print routine: - - fmt.Println(v.Interface()) - -(Why not `fmt.Println(v)`? Because `v` is a `reflect.Value`; we want the concrete value it holds.) Since our value is a `float64`, we can even use a floating-point format if we want: - - fmt.Printf("value is %7.1e\n", v.Interface()) - -and get in this case - - 3.4e+00 - -Again, there's no need to type-assert the result of `v.Interface()` to `float64`; the empty interface value has the concrete value's type information inside and `Printf` will recover it. - -In short, the `Interface` method is the inverse of the `ValueOf` function, except that its result is always of static type `interface{}`. - -Reiterating: Reflection goes from interface values to reflection objects and back again. - -* The third law of reflection - -* 3. To modify a reflection object, the value must be settable. - -The third law is the most subtle and confusing, but it's easy enough to understand if we start from first principles. - -Here is some code that does not work, but is worth studying. - - var x float64 = 3.4 - v := reflect.ValueOf(x) - v.SetFloat(7.1) // Error: will panic. - -If you run this code, it will panic with the cryptic message - - panic: reflect.Value.SetFloat using unaddressable value - -The problem is not that the value `7.1` is not addressable; it's that `v` is not settable. Settability is a property of a reflection `Value`, and not all reflection `Values` have it. - -The `CanSet` method of `Value` reports the settability of a `Value`; in our case, - - var x float64 = 3.4 - v := reflect.ValueOf(x) - fmt.Println("settability of v:", v.CanSet()) - -prints - - settability of v: false - -It is an error to call a `Set` method on an non-settable `Value`. But what is settability? - -Settability is a bit like addressability, but stricter. It's the property that a reflection object can modify the actual storage that was used to create the reflection object. Settability is determined by whether the reflection object holds the original item. When we say - - var x float64 = 3.4 - v := reflect.ValueOf(x) - -we pass a copy of `x` to `reflect.ValueOf`, so the interface value created as the argument to `reflect.ValueOf` is a copy of `x`, not `x` itself. Thus, if the statement - - v.SetFloat(7.1) - -were allowed to succeed, it would not update `x`, even though `v` looks like it was created from `x`. Instead, it would update the copy of `x` stored inside the reflection value and `x` itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. - -If this seems bizarre, it's not. It's actually a familiar situation in unusual garb. Think of passing `x` to a function: - - f(x) - -We would not expect `f` to be able to modify `x` because we passed a copy of `x`'s value, not `x` itself. If we want `f` to modify `x` directly we must pass our function the address of `x` (that is, a pointer to `x`): - - f(&x) - -This is straightforward and familiar, and reflection works the same way. If we want to modify `x` by reflection, we must give the reflection library a pointer to the value we want to modify. - -Let's do that. First we initialize `x` as usual and then create a reflection value that points to it, called `p`. - - var x float64 = 3.4 - p := reflect.ValueOf(&x) // Note: take the address of x. - fmt.Println("type of p:", p.Type()) - fmt.Println("settability of p:", p.CanSet()) - -The output so far is - - type of p: *float64 - settability of p: false - -The reflection object `p` isn't settable, but it's not `p` we want to set, it's (in effect) `*p`. To get to what `p` points to, we call the `Elem` method of `Value`, which indirects through the pointer, and save the result in a reflection `Value` called `v`: - - v := p.Elem() - fmt.Println("settability of v:", v.CanSet()) - -Now `v` is a settable reflection object, as the output demonstrates, - - settability of v: true - -and since it represents `x`, we are finally able to use `v.SetFloat` to modify the value of `x`: - - v.SetFloat(7.1) - fmt.Println(v.Interface()) - fmt.Println(x) - -The output, as expected, is - - 7.1 - 7.1 - -Reflection can be hard to understand but it's doing exactly what the language does, albeit through reflection `Types` and `Values` that can disguise what's going on. Just keep in mind that reflection Values need the address of something in order to modify what they represent. - -* Structs - -In our previous example `v` wasn't a pointer itself, it was just derived from one. A common way for this situation to arise is when using reflection to modify the fields of a structure. As long as we have the address of the structure, we can modify its fields. - -Here's a simple example that analyzes a struct value, `t`. We create the reflection object with the address of the struct because we'll want to modify it later. Then we set `typeOfT` to its type and iterate over the fields using straightforward method calls (see [[http://golang.org/pkg/reflect/][package reflect]] for details). Note that we extract the names of the fields from the struct type, but the fields themselves are regular `reflect.Value` objects. - - type T struct { - A int - B string - } - t := T{23, "skidoo"} - s := reflect.ValueOf(&t).Elem() - typeOfT := s.Type() - for i := 0; i < s.NumField(); i++ { - f := s.Field(i) - fmt.Printf("%d: %s %s = %v\n", i, - typeOfT.Field(i).Name, f.Type(), f.Interface()) - } - -The output of this program is - - 0: A int = 23 - 1: B string = skidoo - -There's one more point about settability introduced in passing here: the field names of `T` are upper case (exported) because only exported fields of a struct are settable. - -Because `s` contains a settable reflection object, we can modify the fields of the structure. - - s.Field(0).SetInt(77) - s.Field(1).SetString("Sunset Strip") - fmt.Println("t is now", t) - -And here's the result: - - t is now {77 Sunset Strip} - -If we modified the program so that `s` was created from `t`, not `&t`, the calls to `SetInt` and `SetString` would fail as the fields of `t` would not be settable. - -* Conclusion - -Here again are the laws of reflection: - -- Reflection goes from interface value to reflection object. - -- Reflection goes from reflection object to interface value. - -- To modify a reflection object, the value must be settable. - -Once you understand these laws reflection in Go becomes much easier to use, although it remains subtle. It's a powerful tool that should be used with care and avoided unless strictly necessary. - -There's plenty more to reflection that we haven't covered — sending and receiving on channels, allocating memory, using slices and maps, calling methods and functions — but this post is long enough. We'll cover some of those topics in a later article. diff --git a/content/learn-go-from-your-browser.article b/content/learn-go-from-your-browser.article deleted file mode 100644 index e068eb55..00000000 --- a/content/learn-go-from-your-browser.article +++ /dev/null @@ -1,15 +0,0 @@ -Learn Go from your browser -4 Oct 2011 -Tags: playground, tour - -Andrew Gerrand - -* Introduction - -We are excited to announce [[http://go-tour.appspot.com/][A Tour of Go]], a guided tour of the Go programming language you can run from your browser. - -The tour is hands-on, demonstrating the language through code samples that you can modify, compile, and run from the tour itself. (The technology behind the [[http://golang.org/doc/play/][Go Playground]] does the work.) - -The tour has three sections. The first section covers basic syntax and data structures; the second discusses methods and interfaces; and the third introduces Go's concurrency primitives. Each section concludes with a few exercises so you can practice what you've learned. - -So, what are you waiting for? [[http://go-tour.appspot.com/][Get started now!]] diff --git a/content/new-talk-and-tutorials.article b/content/new-talk-and-tutorials.article deleted file mode 100644 index 608c47a4..00000000 --- a/content/new-talk-and-tutorials.article +++ /dev/null @@ -1,12 +0,0 @@ -New Talk and Tutorials -5 May 2010 - -Andrew Gerrand - -* Introduction - -Rob Pike recently gave a talk at Stanford's [[http://www.stanford.edu/class/ee380/][Computer Systems Colloquium]] (EE380). Titled [[http://www.stanford.edu/class/ee380/Abstracts/100428.html][_Another_Go_at_Language_Design_]], the presentation gives an overview of the itches Go was built to scratch, and how Go addresses those problems. You can view [[http://stanford-online.stanford.edu/courses/ee380/100428-ee380-300.asx][a video stream of the talk]], and [[http://www.stanford.edu/class/ee380/Abstracts/100428-pike-stanford.pdf][download the slides]]. - -Last week's release included a code lab, [[http://golang.org/doc/codelab/wiki/][Writing Web Applications]], that details the construction of a simple wiki program. It is a practical introduction to some fundamental Go concepts, and the first of a series of Go code labs. - -Lastly, we are often asked "How do Go packages work?" It's easier to show than to explain, so I put together a [[http://www.youtube.com/watch?v=jDWBJOXs_iI][Go Packages screen cast]] that demonstrates the process of writing, building, installing, and redistributing Go packages. I hope to post more of these covering a variety of Go programming topics to the [[http://youtube.com/gocoding][gocoding YouTube channel]] in the near future. diff --git a/content/normalization.article b/content/normalization.article deleted file mode 100644 index 1763fcb9..00000000 --- a/content/normalization.article +++ /dev/null @@ -1,283 +0,0 @@ -Text normalization in Go -26 Nov 2013 -Tags: strings, bytes, runes, characters - -Marcel van Lohuizen - -* Introduction - -An earlier [[http://blog.golang.org/strings][post]] talked about strings, bytes -and characters in Go. I've been working on various packages for multilingual -text processing for the go.text repository. Several of these packages deserve a -separate blog post, but today I want to focus on -[[http://godoc.org/code.google.com/p/go.text/unicode/norm][go.text/unicode/norm]], -which handles normalization, a topic touched in the -[[http://blog.golang.org/strings][strings article]] and the subject of this -post. Normalization works at a higher level of abstraction than raw bytes. - -To learn pretty much everything you ever wanted to know about normalization -(and then some), [[http://unicode.org/reports/tr15/][Annex 15 of the Unicode Standard]] -is a good read. A more approachable article is the corresponding -[[http://en.wikipedia.org/wiki/Unicode_equivalence][Wikipedia page]]. Here we -focus on how normalization relates to Go. - -* What is normalization? - -There are often several ways to represent the same string. For example, an é -(e-acute) can be represented in a string as a single rune ("\u00e9") or an 'e' -followed by an acute accent ("e\u0301"). According to the Unicode standard, -these two are "canonically equivalent" and should be treated as equal. - -Using a byte-to-byte comparison to determine equality would clearly not give -the right result for these two strings. Unicode defines a set of normal forms -such that if two strings are canonically equivalent and are normalized to the -same normal form, their byte representations are the same. - -Unicode also defines a "compatibility equivalence" to equate characters that -represent the same characters, but may have a different visual appearance. For -example, the superscript digit '⁹' and the regular digit '9' are equivalent in -this form. - -For each of these two equivalence forms, Unicode defines a composing and -decomposing form. The former replaces runes that can combine into a single rune -with this single rune. The latter breaks runes apart into their components. -This table shows the names, all starting with NF, by which the Unicode -Consortium identifies these forms: - -.html normalization/table1.html - -* Go's approach to normalization - -As mentioned in the strings blog post, Go does not guarantee that characters in -a string are normalized. However, the go.text packages can compensate. For -example, the -[[http://godoc.org/code.google.com/p/go.text/collate][collate]] package, which -can sort strings in a language-specific way, works correctly even with -unnormalized strings. The packages in go.text do not always require normalized -input, but in general normalization may be necessary for consistent results. - -Normalization isn't free but it is fast, particularly for collation and -searching or if a string is either in NFD or in NFC and can be converted to NFD -by decomposing without reordering its bytes. In practice, -[[http://www.macchiato.com/unicode/nfc-faq#TOC-How-much-text-is-already-NFC-][99.98%]] of -the web's HTML page content is in NFC form (not counting markup, in which case -it would be more). By far most NFC can be decomposed to NFD without the need -for reordering (which requires allocation). Also, it is efficient to detect -when reordering is necessary, so we can save time by doing it only for the rare -segments that need it. - -To make things even better, the collation package typically does not use the -norm package directly, but instead uses the norm package to interleave -normalization information with its own tables. Interleaving the two problems -allows for reordering and normalization on the fly with almost no impact on -performance. The cost of on-the-fly normalization is compensated by not having -to normalize text beforehand and ensuring that the normal form is maintained -upon edits. The latter can be tricky. For instance, the result of concatenating -two NFC-normalized strings is not guaranteed to be in NFC. - -Of course, we can also avoid the overhead outright if we know in advance that a -string is already normalized, which is often the case. - -* Why bother? - -After all this discussion about avoiding normalization, you might ask why it's -worth worrying about at all. The reason is that there are cases where -normalization is required and it is important to understand what those are, and -in turn how to do it correctly. - -Before discussing those, we must first clarify the concept of 'character'. - -* What is a character? - -As was mentioned in the strings blog post, characters can span multiple runes. -For example, an 'e' and '◌́' (acute "\u0301") can combine to form 'é' ("e\u0301" -in NFD).  Together these two runes are one character. The definition of a -character may vary depending on the application. For normalization we will -define it as a sequence of runes that starts with a starter, a rune that does -not modify or combine backwards with any other rune, followed by possibly empty -sequence of non-starters, that is, runes that do (typically accents). The -normalization algorithm processes one character at at time. - -Theoretically, there is no bound to the number of runes that can make up a -Unicode character. In fact, there are no restrictions on the number of -modifiers that can follow a character and a modifier may be repeated, or -stacked. Ever seen an 'e' with three acutes? Here you go: 'é́́'. That is a -perfectly valid 4-rune character according to the standard. - -As a consequence, even at the lowest level, text needs to be processed in -increments of unbounded chunk sizes. This is especially awkward with a -streaming approach to text processing, as used by Go's standard Reader and -Writer interfaces, as that model potentially requires any intermediate buffers -to have unbounded size as well. Also, a straightforward implementation of -normalization will have a O(n²) running time. - -There are really no meaningful interpretations for such large sequences of -modifiers for practical applications. Unicode defines a Stream-Safe Text -format, which allows capping the number of modifiers (non-starters) to at most -30, more than enough for any practical purpose. Subsequent modifiers will be -placed after a freshly inserted Combining Grapheme Joiner (CGJ or U+034F). Go -adopts this approach for all normalization algorithms. This decision gives up a -little conformance but gains a little safety. - -* Writing in normal form - -Even if you don't need to normalize text within your Go code, you might still -want to do so when communicating to the outside world. For example, normalizing -to NFC might compact your text, making it cheaper to send down a wire. For some -languages, like Korean, the savings can be substantial. Also, some external -APIs might expect text in a certain normal form. Or you might just want to fit -in and output your text as NFC like the rest of the world. - -To write your text as NFC, use the -[[http://godoc.org/code.google.com/p/go.text/unicode/norm][unicode/norm]] package -to wrap your `io.Writer` of choice: - - wc := norm.NFC.Writer(w) - defer wc.Close() - // write as before... - -If you have a small string and want to do a quick conversion, you can use this -simpler form: - - norm.NFC.Bytes(b) - -Package norm provides various other methods for normalizing text. -Pick the one that suits your needs best. - -* Catching look-alikes - -Can you tell the difference between 'K' ("\u004B") and 'K' (Kelvin sign -"\u212A") or 'Ω' ("\u03a9") and 'Ω' (Ohm sign "\u2126")? It is easy to overlook -the sometimes minute differences between variants of the same underlying -character. It is generally a good idea to disallow such variants in identifiers -or anything where deceiving users with such look-alikes can pose a security -hazard. - -The compatibility normal forms, NFKC and NFKD, will map many visually nearly -identical forms to a single value. Note that it will not do so when two symbols -look alike, but are really from two different alphabets. For example the Latin -'o', Greek 'ο', and Cyrillic 'о' are still different characters as defined by -these forms. - -* Correct text modifications - -The norm package might also come to the rescue when one needs to modify text. -Consider a case where you want to search and replace the word "cafe" with its -plural form "cafes".  A code snippet could look like this. - - s := "We went to eat at multiple cafe" - cafe := "cafe" - if p := strings.Index(s, cafe); p != -1 { - p += len(cafe) - s = s[:p] + "s" + s[p:] - } - fmt.Println(s) - -This prints "We went to eat at multiple cafes" as desired and expected. Now -consider our text contains the French spelling "café" in NFD form: - - s := "We went to eat at multiple cafe\u0301" - -Using the same code from above, the plural "s" would still be inserted after -the 'e', but before the acute, resulting in  "We went to eat at multiple -cafeś".  This behavior is undesirable. - -The problem is that the code does not respect the boundaries between multi-rune -characters and inserts a rune in the middle of a character.  Using the norm -package, we can rewrite this piece of code as follows: - - s := "We went to eat at multiple cafe\u0301" - cafe := "cafe" - if p := strings.Index(s, cafe); p != -1 { - p += len(cafe) - if bp := norm.FirstBoundary(s[p:]); bp > 0 { - p += bp - } - s = s[:p] + "s" + s[p:] - } - fmt.Println(s) - -This may be a contrived example, but the gist should be clear. Be mindful of -the fact that characters can span multiple runes. Generally these kinds of -problems can be avoided by using search functionality that respects character -boundaries (such as the planned go.text/search package.) - -* Iteration - -Another tool provided by the norm package that may help dealing with character -boundaries is its iterator, -[[http://godoc.org/code.google.com/p/go.text/unicode/norm#Iter][`norm.Iter`]]. -It iterates over characters one at a time in the normal form of choice. - -* Performing magic - -As mentioned earlier, most text is in NFC form, where base characters and -modifiers are combined into a single rune whenever possible.  For the purpose -of analyzing characters, it is often easier to handle runes after decomposition -into their smallest components. This is where the NFD form comes in handy. For -example, the following piece of code creates a `transform.Transformer` that -decomposes text into its smallest parts, removes all accents, and then -recomposes the text into NFC: - - import ( - "unicode" - - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" - ) - - isMn := func(r rune) bool { - return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks - } - t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) - -The resulting `Transformer` can be used to remove accents from an `io.Reader` -of choice as follows: - - r = transform.NewReader(r, t) - // read as before ... - -This will, for example, convert any mention of "cafés" in the text to "cafes", -regardless of the normal form in which the original text was encoded. - -* Normalization info - -As mentioned earlier, some packages precompute normalizations into their tables -to minimize the need for normalization at run time. The type `norm.Properties` -provides access to the per-rune information needed by these packages, most -notably the Canonical Combining Class and decomposition information. Read the -[[http://godoc.org/code.google.com/p/go.text/unicode/norm/#Properties][documentation]] -for this type if you want to dig deeper. - -* Performance - -To give an idea of the performance of normalization, we compare it against the -performance of strings.ToLower. The sample in the first row is both lowercase -and NFC and can in every case be returned as is. The second sample is neither -and requires writing a new version. - -.html normalization/table2.html - -The column with the results for the iterator shows both the measurement with -and without initialization of the iterator, which contain buffers that don't -need to be reinitialized upon reuse. - -As you can see, detecting whether a string is normalized can be quite -efficient. A lot of the cost of normalizing in the second row is for the -initialization of buffers, the cost of which is amortized when one is -processing larger strings. As it turns out, these buffers are rarely needed, so -we may change the implementation at some point to speed up the common case for -small strings even further. - -* Conclusion - -If you're dealing with text inside Go, you generally do not have to use the -unicode/norm package to normalize your text. The package may still be useful -for things like ensuring that strings are normalized before sending them out or -to do advanced text manipulation. - -This article briefly mentioned the existence of other go.text packages as well -as multilingual text processing and it may have raised more questions than it -has given answers. The discussion of these topics, however, will have to wait -until another day. - diff --git a/content/normalization/table1.html b/content/normalization/table1.html deleted file mode 100644 index 824cbdf9..00000000 --- a/content/normalization/table1.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - -
 ComposingDecomposing
Canonical equivalenceNFCNFD
Compatibility equivalenceNFKCNFKD
diff --git a/content/normalization/table2.html b/content/normalization/table2.html deleted file mode 100644 index 2ecf2940..00000000 --- a/content/normalization/table2.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - -
InputToLowerNFC AppendNFC TransformNFC Iter
nörmalization199 ns137 ns133 ns251 ns (621 ns)
No\u0308rmalization427 ns836 ns845 ns573 ns (948 ns)
diff --git a/content/open-source.article b/content/open-source.article deleted file mode 100644 index 2fa797ae..00000000 --- a/content/open-source.article +++ /dev/null @@ -1,1120 +0,0 @@ -Go, Open Source, Community -08 Jul 2015 -Tags: community - -Russ Cox - -* Welcome - -[This is the text of my opening keynote at Gophercon 2015. -[[https://www.youtube.com/watch?v=XvZOdpd_9tc][The video is available here]].] - -Thank you all for traveling to Denver to be here, -and thank you to everyone watching on video. -If this is your first Gophercon, welcome. -If you were here last year, welcome back. -Thank you to the organizers -for all the work it takes -to make a conference like this happen. -I am thrilled to be here and to be able to talk to all of you. - -I am the tech lead for the Go project -and the Go team at Google. -I share that role with Rob Pike. -In that role, I spend a lot of time thinking about -the overall Go open source project, -in particular the way it runs, -what it means to be open source, -and the interaction between -contributors inside and outside Google. -Today I want to share with you -how I see the Go project as a whole -and then based on that explain -how I see the Go open source project -evolving. - -* Why Go? - -To get started, -we have to go back to the beginning. -Why did we start working on Go? - -Go is an attempt to make programmers more productive. -We wanted to improve the software development process -at Google, -but the problems Google has -are not unique to Google. - -There were two overarching goals. - -The first goal is to make a better language -to meet the challenges of scalable concurrency. -By scalable concurrency I mean -software that deals with many concerns simultaneously, -such as coordinating a thousand back end servers -by sending network traffic back and forth. - -Today, that kind of software has a shorter name: -we call it cloud software. -It's fair to say that Go was designed for the cloud -before clouds ran software. - -The larger goal is to make a better environment -to meet the challenges of scalable software development, -software worked on and used by many people, -with limited coordination between them, -and maintained for years. -At Google we have thousands of engineers -writing and sharing their code with each other, -trying to get their work done, -reusing the work of others as much as possible, -and working in a code base with a history -dating back over ten years. -Engineers often work on or at least look at -code originally written by someone else, -or that they wrote years ago, -which often amounts to the same thing. - -That situation inside Google -has a lot in common with -large scale, modern open source development -as practiced on sites like GitHub. -Because of this, -Go is a great fit for open source projects, -helping them accept and manage -contributions from a large community -over a long period of time. - -I believe much of Go's success is explained by the fact that -Go is a great fit for cloud software, -Go is a great fit for open source projects, -and, serendipitously, both of those are -growing in popularity and importance -in the software industry. - -Other people have made similar observations. -Here are two. -Last year, on RedMonk.com, Donnie Berkholz -wrote about -“[[http://redmonk.com/dberkholz/2014/03/18/go-the-emerging-language-of-cloud-infrastructure/][Go as the emerging language of cloud infrastructure]],” -observing that -“[Go's] marquee projects ... are cloud-centric or otherwise -made for dealing with distributed systems -or transient environments.” - -This year, on Texlution.com, the author -wrote an article titled -“[[https://texlution.com/post/why-go-is-doomed-to-succeed/][Why Golang is doomed to succeed]],” -pointing out that this focus on large-scale development -was possibly even better suited to open source than -to Google itself: “This open source fitness is why I think -you are about to see more and more Go around ...” - -* The Go Balance - -How does Go accomplish those things? - -How does it make scalable concurrency -and scalable software development easier? - -Most people answer this question by talking about -channels and goroutines, and interfaces, and fast builds, -and the go command, and good tool support. -Those are all important parts of the answer, -but I think there is a broader idea behind them. - -I think of that idea as Go's balance. -There are competing concerns in any software design, -and there is a very natural tendency to try to solve -all the problems you foresee. -In Go, we have explicitly tried not to solve everything. -Instead, we've tried to do just enough that you can build -your own custom solutions easily. - -The way I would summarize Go's chosen balance is this: *Do*Less.*Enable*More.* - -Do less, but enable more. - -Go can't do everything. -We shouldn't try. -But if we work at it, -Go can probably do -a few things well. -If we select those things carefully, -we can lay a foundation -on which developers can _easily_ build -the solutions and tools they need, -and ideally can interoperate with -the solutions and tools built by others. - -** Examples - -Let me illustrate this with some examples. - -First, the size of the Go language itself. -We worked hard to put in as few concepts as possible, -to avoid the problem of mutually incomprehensible dialects -forming in different parts of a large developer community. -No idea went into Go until -it had been simplified to its essence -and then had clear benefits -that justified the complexity being added. - -In general, if we have 100 things -we want Go to do well, -we can't make 100 separate changes. -Instead, we try to research and understand -the design space -and then identify a few changes -that work well together -and that enable maybe 90 of those things. -We're willing to sacrifice the remaining 10 -to avoid bloating the language, -to avoid adding complexity -only to address specific use cases -that seem important today -but might be gone tomorrow. - -Keeping the language small -enables more important goals. -Being small makes Go -easier to learn, -easier to understand, -easier to implement, -easier to reimplement, -easier to debug, -easier to adjust, -and easier to evolve. -Doing less enables more. - -I should point out that -this means we say no -to a lot of other people's ideas, -but I assure you -we've said no -to even more of our own ideas. - -Next, channels and goroutines. -How should we structure and coordinate -concurrent and parallel computations? -Mutexes and condition variables are very general -but so low-level that they're difficult to use correctly. -Parallel execution frameworks like OpenMP are so high-level -that they can only be used to solve a narrow range of problems. -Channels and goroutines sit between these two extremes. -By themselves, they aren't a solution to much. -But they are powerful enough to be easily arranged -to enable solutions to many common problems -in concurrent software. -Doing less—really doing just enough—enables more. - -Next, types and interfaces. -Having static types enables useful compile-time checking, -something lacking in dynamically-typed languages -like Python or Ruby. -At the same time, -Go's static typing avoids -much of the repetition -of traditional statically typed languages, -making it feel more lightweight, -more like the dynamically-typed languages. -This was one of the first things people noticed, -and many of Go's early adopters came from -dynamically-typed languages. - -Go's interfaces are a key part of that. -In particular, -omitting the ``implements'' declarations -of Java or other languages with static hierarchy -makes interfaces lighter weight and more flexible. -Not having that rigid hierarchy -enables idioms such as test interfaces that describe -existing, unrelated production implementations. -Doing less enables more. - -Next, testing and benchmarking. -Is there any shortage of testing -and benchmarking frameworks in most languages? -Is there any agreement between them? - -Go's testing package is not meant -to address every possible facet of these topics. -Instead, it is meant to provide -the basic concepts necessary -for most higher-level tooling. -Packages have test cases that pass, fail, or are skipped. -Packages have benchmarks that run and can be measured -by various metrics. - -Doing less here is an attempt -to reduce these concepts to their essence, -to create a shared vocabulary -so that richer tools can interoperate. -That agreement enables higher-level testing software -like Miki Tebeka's go2xunit converter, -or the benchcmp and benchstat -benchmark analysis tools. - -Because there _is_ agreement -about the representation of the basic concepts, -these higher-level tools work for all Go packages, -not just ones that make the effort to opt in, -and they interoperate with each other, -in that using, say, go2xunit -does not preclude also using benchstat, -the way it would if these tools were, say, -plugins for competing testing frameworks. -Doing less enables more. - -Next, refactoring and program analysis. -Because Go is for large code bases, -we knew it would need to support automatic -maintenance and updating of source code. -We also knew that this topic was too large -to build in directly. -But we knew one thing that we had to do. -In our experience attempting -automated program changes in other settings, -the most significant barrier we hit -was actually writing the modified program out -in a format that developers can accept. - -In other languages, -it's common for different teams to use -different formatting conventions. -If an edit by a program uses the wrong convention, -it either writes a section of the source file that looks nothing -like the rest of the file, or it reformats the entire file, -causing unnecessary and unwanted diffs. - -Go does not have this problem. -We designed the language to make gofmt possible, -we worked hard -to make gofmt's formatting acceptable -for all Go programs, -and we made sure gofmt was there -from day one of the original public release. -Gofmt imposes such uniformity that -automated changes blend into the rest of the file. -You can't tell whether a particular change -was made by a person or a computer. -We didn't build explicit refactoring support. -Establishing an agreed-upon formatting algorithm -was enough of a shared base -for independent tools to develop and to interoperate. -Gofmt enabled gofix, goimports, eg, and other tools. -I believe the work here is only just getting started. -Even more can be done. - -Last, building and sharing software. -In the run up to Go 1, we built goinstall, -which became what we all know as "go get". -That tool defined a standard zero-configuration way -to resolve import paths on sites like github.com, -and later a way to resolve paths on other sites -by making HTTP requests. -This agreed-upon resolution algorithm -enabled other tools that work in terms of those paths, -most notably Gary Burd's creation of godoc.org. -In case you haven't used it, -you go to godoc.org/the-import-path -for any valid "go get" import path, -and the web site will fetch the code -and show you the documentation for it. -A nice side effect of this has been that -godoc.org serves as a rough master list -of the Go packages publicly available. -All we did was give import paths a clear meaning. -Do less, enable more. - -You'll notice that many of these tooling examples -are about establishing a shared convention. -Sometimes people refer to this as Go being “opinionated,” -but there's something deeper going on. -Agreeing to the limitations -of a shared convention -is a way to enable -a broad class of tools that interoperate, -because they all speak the same base language. -This is a very effective way -to do less but enable more. -Specifically, in many cases -we can do the minimum required -to establish a shared understanding -of a particular concept, like remote imports, -or the proper formatting of a source file, -and thereby enable -the creation of packages and tools -that work together -because they all agree -about those core details. - -I'm going to return to that idea later. - -* Why is Go open source? - -But first, as I said earlier, -I want to explain how I see -the balance of Do Less and Enable More -guiding our work -on the broader -Go open source project. -To do that, I need to start with -why Go is open source at all. - -Google pays me and others to work on Go, because, -if Google's programmers are more productive, -Google can build products faster, -maintain them more easily, -and so on. -But why open source Go? -Why should Google share this benefit with the world? - -Of course, many of us -worked on open source projects before Go, -and we naturally wanted Go -to be part of that open source world. -But our preferences are not a business justification. -The business justification is that -Go is open source -because that's the only way -that Go can succeed. -We, the team that built Go within Google, -knew this from day one. -We knew that Go had to be made available -to as many people as possible -for it to succeed. - -Closed languages die. - -A language needs large, broad communities. - -A language needs lots of people writing lots of software, -so that when you need a particular tool or library, -there's a good chance it has already been written, -by someone who knows the topic better than you, -and who spent more time than you have to make it great. - -A language needs lots of people reporting bugs, -so that problems are identified and fixed quickly. -Because of the much larger user base, -the Go compilers are much more robust and spec-compliant -than the Plan 9 C compilers they're loosely based on ever were. - -A language needs lots of people using it -for lots of different purposes, -so that the language doesn't overfit to one use case -and end up useless when the technology landscape changes. - -A language needs lots of people who want to learn it, -so that there is a market for people to write books -or teach courses, -or run conferences like this one. - -None of this could have happened -if Go had stayed within Google. -Go would have suffocated inside Google, -or inside any single company -or closed environment. - -Fundamentally, -Go must be open, -and Go needs you. -Go can't succeed without all of you, -without all the people using Go -for all different kinds of projects -all over the world. - -In turn, the Go team at Google -could never be large enough -to support the entire Go community. -To keep scaling, -we -need to enable all this ``more'' -while doing less. -Open source is a huge part of that. - -* Go's open source - -What does open source mean? -The minimum requirement is to open the source code, -making it available under an open source license, -and we've done that. - -But we also opened our development process: -since announcing Go, -we've done all our development in public, -on public mailing lists open to all. -We accept and review -source code contributions from anyone. -The process is the same -whether you work for Google or not. -We maintain our bug tracker in public, -we discuss and develop proposals for changes in public, -and we work toward releases in public. -The public source tree is the authoritative copy. -Changes happen there first. -They are only brought into -Google's internal source tree later. -For Go, being open source means -that this is a collective effort -that extends beyond Google, open to all. - -Any open source project starts with a few people, -often just one, but with Go it was three: -Robert Griesemer, Rob Pike, and Ken Thompson. -They had a vision of -what they wanted Go to be, -what they thought Go could do better -than existing languages, and -Robert will talk more about that tomorrow morning. -I was the next person to join the team, -and then Ian Taylor, -and then, one by one, -we've ended up where we are today, -with hundreds of contributors. - -Thank You -to the many people who have contributed -code -or ideas -or bug reports -to the Go project so far. -We tried to list everyone we could -in our space in the program today. -If your name is not there, -I apologize, -but thank you. - -I believe -the hundreds of contributors so far -are working toward a shared vision -of what Go can be. -It's hard to put words to these things, -but I did my best -to explain one part of the vision -earlier: -Do Less, Enable More. - -* Google's role - -A natural question is: -What is the role -of the Go team at Google, -compared to other contributors? -I believe that role -has changed over time, -and it continues to change. -The general trend is that -over time -the Go team at Google -should be doing less -and enabling more. - -In the very early days, -before Go was known to the public, -the Go team at Google -was obviously working by itself. -We wrote the first draft of everything: -the specification, -the compiler, -the runtime, -the standard library. - -Once Go was open sourced, though, -our role began to change. -The most important thing -we needed to do -was communicate our vision for Go. -That's difficult, -and we're still working at it.. -The initial implementation -was an important way -to communicate that vision, -as was the development work we led -that resulted in Go 1, -and the various blog posts, -and articles, -and talks we've published. - -But as Rob said at Gophercon last year, -"the language is done." -Now we need to see how it works, -to see how people use it, -to see what people build. -The focus now is on -expanding the kind of work -that Go can help with. - -Google's primarily role is now -to enable the community, -to coordinate, -to make sure changes work well together, -and to keep Go true to the original vision. - -Google's primary role is: -Do Less. Enable More. - -I mentioned earlier -that we'd rather have a small number of features -that enable, say, 90% of the target use cases, -and avoid the orders of magnitude -more features necessary -to reach 99 or 100%. -We've been successful in applying that strategy -to the areas of software that we know well. -But if Go is to become useful in many new domains, -we need experts in those areas -to bring their expertise -to our discussions, -so that together -we can design small adjustments -that enable many new applications for Go. - -This shift applies not just to design -but also to development. -The role of the Go team at Google -continues to shift -more to one of guidance -and less of pure development. -I certainly spend much more time -doing code reviews than writing code, -more time processing bug reports -than filing bug reports myself. -We need to do less and enable more. - -As design and development shift -to the broader Go community, -one of the most important things -we -the original authors of Go -can offer -is consistency of vision, -to help keep Go -Go. -The balance that we must strike -is certainly subjective. -For example, a mechanism for extensible syntax -would be a way to -enable more -ways to write Go code, -but that would run counter to our goal -of having a consistent language -without different dialects. - -We have to say no sometimes, -perhaps more than in other language communities, -but when we do, -we aim to do so -constructively and respectfully, -to take that as an opportunity -to clarify the vision for Go. - -Of course, it's not all coordination and vision. -Google still funds Go development work. -Rick Hudson is going to talk later today -about his work on reducing garbage collector latency, -and Hana Kim is going to talk tomorrow -about her work on bringing Go to mobile devices. -But I want to make clear that, -as much as possible, -we aim to treat -development funded by Google -as equal to -development funded by other companies -or contributed by individuals using their spare time. -We do this because we don't know -where the next great idea will come from. -Everyone contributing to Go -should have the opportunity to be heard. - -** Examples - -I want to share some evidence for this claim -that, over time, -the original Go team at Google -is focusing more on -coordination than direct development. - -First, the sources of funding -for Go development are expanding. -Before the open source release, -obviously Google paid for all Go development. -After the open source release, -many individuals started contributing their time, -and we've slowly but steadily -been growing the number of contributors -supported by other companies -to work on Go at least part-time, -especially as it relates to -making Go more useful for those companies. -Today, that list includes -Canonical, Dropbox, Intel, Oracle, and others. -And of course Gophercon and the other -regional Go conferences are organized -entirely by people outside Google, -and they have many corporate sponsors -besides Google. - -Second, the conceptual depth -of Go development -done outside the original team -is expanding. - -Immediately after the open source release, -one of the first large contributions -was the port to Microsoft Windows, -started by Hector Chu -and completed by Alex Brainman and others. -More contributors ported Go -to other operating systems. -Even more contributors -rewrote most of our numeric code -to be faster or more precise or both. -These were all important contributions, -and very much appreciated, -but -for the most part -they did not involve new designs. - -More recently, -a group of contributors led by Aram Hăvărneanu -ported Go to the ARM 64 architecture, -This was the first architecture port -by contributors outside Google. -This is significant, because -in general -support for a new architecture -requires more design work -than support for a new operating system. -There is more variation between architectures -than between operating systems. - -Another example is the introduction -over the past few releases -of preliminary support -for building Go programs using shared libraries. -This feature is important for many Linux distributions -but not as important for Google, -because we deploy static binaries. -We have been helping guide the overall strategy, -but most of the design -and nearly all of the implementation -has been done by contributors outside Google, -especially Michael Hudson-Doyle. - -My last example is the go command's -approach to vendoring. -I define vendoring as -copying source code for external dependencies -into your tree -to make sure that they doesn't disappear -or change underfoot. - -Vendoring is not a problem Google suffers, -at least not the way the rest of the world does. -We copy open source libraries we want to use -into our shared source tree, -record what version we copied, -and only update the copy -when there is a need to do so. -We have a rule -that there can only be one version -of a particular library in the source tree, -and it's the job of whoever wants to upgrade that library -to make sure it keeps working as expected -by the Google code that depends on it. -None of this happens often. -This is the lazy approach to vendoring. - -In contrast, most projects outside Google -take a more eager approach, -importing and updating code -using automated tools -and making sure that they are -always using the latest versions. - -Because Google has relatively little experience -with this vendoring problem, -we left it to users outside Google to develop solutions. -Over the past five years, -people have built a series of tools. -The main ones in use today are -Keith Rarick's godep, -Owen Ou's nut, -and the gb-vendor plugin for Dave Cheney's gb, - -There are two problems with the current situation. -The first is that these tools -are not compatible -out of the box -with the go command's "go get". -The second is that the tools -are not even compatible with each other. -Both of these problems -fragment the developer community by tool. - -Last fall, we started a public design discussion -to try to build consensus on -some basics about -how these tools all operate, -so that they can work alongside "go get" -and each other. - -Our basic proposal was that all tools agree -on the approach of rewriting import paths during vendoring, -to fit with "go get"'s model, -and also that all tools agree on a file format -describing the source and version of the copied code, -so that the different vendoring tools -can be used together -even by a single project. -If you use one today, -you should still be able to use another tomorrow. - -Finding common ground in this way -was very much in the spirit of Do Less, Enable More. -If we could build consensus -about these basic semantic aspects, -that would enable "go get" and all these tools to interoperate, -and it would enable switching between tools, -the same way that -agreement about how Go programs -are stored in text files -enables the Go compiler and all text editors to interoperate. -So we sent out our proposal for common ground. - -Two things happened. - -First, Daniel Theophanes -started a vendor-spec project on GitHub -with a new proposal -and took over coordination and design -of the spec for vendoring metadata. - -Second, the community spoke -with essentially one voice -to say that -rewriting import paths during vendoring -was not tenable. -Vendoring works much more smoothly -if code can be copied without changes. - -Keith Rarick posted an alternate proposal -for a minimal change to the go command -to support vendoring without rewriting import paths. -Keith's proposal was configuration-free -and fit in well with the rest of the go command's approach. -That proposal will ship -as an experimental feature in Go 1.5 -and likely enabled by default in Go 1.6. -And I believe that the various vendoring tool authors -have agreed to adopt Daniel's spec once it is finalized. - -The result -is that at the next Gophercon -we should have broad interoperability -between vendoring tools and the go command, -and the design to make that happen -was done entirely by contributors -outside the original Go team. - -Not only that, -the Go team's proposal for how to do this -was essentially completely wrong. -The Go community told us that -very clearly. -We took that advice, -and now there's a plan for vendoring support -that I believe -everyone involved is happy with. - -This is also a good example -of our general approach to design. -We try not to make any changes to Go -until we feel there is broad consensus -on a well-understood solution. -For vendoring, -feedback and design -from the Go community -was critical to reaching that point. - -This general trend -toward both code and design -coming from the broader Go community -is important for Go. -You, the broader Go community, -know what is working -and what is not -in the environments where you use Go. -We at Google don't. -More and more, -we will rely on your expertise, -and we will try to help you develop -designs and code -that extend Go to be useful in more settings -and fit well with Go's original vision. -At the same time, -we will continue to wait -for broad consensus -on well-understood solutions. - -This brings me to my last point. - -* Code of Conduct - -I've argued that Go must be open, -and that Go needs your help. - -But in fact Go needs everyone's help. -And everyone isn't here. - -Go needs ideas from as many people as possible. - -To make that a reality, -the Go community needs to be -as inclusive, -welcoming, -helpful, -and respectful as possible. - -The Go community is large enough now that, -instead of assuming that everyone involved -knows what is expected, -I and others believe that it makes sense -to write down those expectations explicitly. -Much like the Go spec -sets expectations for all Go compilers, -we can write a spec -setting expectations for our behavior -in online discussions -and in offline meetings like this one. - -Like any good spec, -it must be general enough -to allow many implementations -but specific enough -that it can identify important problems. -When our behavior doesn't meet the spec, -people can point that out to us, -and we can fix the problem. -At the same time, -it's important to understand that -this kind of spec -cannot be as precise as a language spec. -We must start with the assumption -that we will all be reasonable in applying it. - -This kind of spec -is often referred to as -a Code of Conduct. -Gophercon has one, -which we've all agreed to follow -by being here, -but the Go community does not. -I and others -believe the Go community -needs a Code of Conduct. - -But what should it say? - -I believe -the most important -overall statement we can make -is that -if you want to use or discuss Go, -then you are welcome here, -in our community. -That is the standard -I believe we aspire to. - -If for no other reason -(and, to be clear, there are excellent other reasons), -Go needs as large a community as possible. -To the extent that behavior -limits the size of the community, -it holds Go back. -And behavior can easily -limit the size of the community. - -The tech community in general -and the Go community in particular -is skewed toward people who communicate bluntly. -I don't believe this is fundamental. -I don't believe this is necessary. -But it's especially easy to do -in online discussions like email and IRC, -where plain text is not supplemented -by the other cues and signals we have -in face-to-face interactions. - -For example, I have learned -that when I am pressed for time -I tend to write fewer words, -with the end result that -my emails seem not just hurried -but blunt, impatient, even dismissive. -That's not how I feel, -but it's how I can come across, -and that impression can be enough -to make people think twice -about using or contributing -to Go. -I realized I was doing this -when some Go contributors -sent me private email to let me know. -Now, when I am pressed for time, -I pay extra attention to what I'm writing, -and I often write more than I naturally would, -to make sure -I'm sending the message I intend. - -I believe -that correcting the parts -of our everyday interactions, -intended or not, -that drive away potential users and contributors -is one of the most important things -we can all do -to make sure the Go community -continues to grow. -A good Code of Conduct can help us do that. - -We have no experience writing a Code of Conduct, -so we have been reading existing ones, -and we will probably adopt an existing one, -perhaps with minor adjustments. -The one I like the most is the Django Code of Conduct, -which originated with another project called SpeakUp! -It is structured as an elaboration of a list of -reminders for everyday interaction. - -"Be friendly and patient. -Be welcoming. -Be considerate. -Be respectful. -Be careful in the words that you choose. -When we disagree, try to understand why." - -I believe this captures the tone we want to set, -the message we want to send, -the environment we want to create -for new contributors. -I certainly want to be -friendly, -patient, -welcoming, -considerate, -and respectful. -I won't get it exactly right all the time, -and I would welcome a helpful note -if I'm not living up to that. -I believe most of us -feel the same way. - -I haven't mentioned -active exclusion based on -or disproportionately affecting -race, gender, disability, -or other personal characteristics, -and I haven't mentioned harassment. -For me, -it follows from what I just said -that exclusionary behavior -or explicit harassment -is absolutely unacceptable, -online and offline. -Every Code of Conduct says this explicitly, -and I expect that ours will too. -But I believe the SpeakUp! reminders -about everyday interactions -are an equally important statement. -I believe that -setting a high standard -for those everyday interactions -makes extreme behavior -that much clearer -and easier to deal with. - -I have no doubts that -the Go community can be -one of the most -friendly, -welcoming, -considerate, -and -respectful communities -in the tech industry. -We can make that happen, -and it will be -a benefit and credit to us all. - -Andrew Gerrand -has been leading the effort -to adopt an appropriate Code of Conduct -for the Go community. -If you have suggestions, -or concerns, -or experience with Codes of Conduct, -or want to be involved, -please find Andrew or me -during the conference. -If you'll still be here on Friday, -Andrew and I are going to block off -some time for Code of Conduct discussions -during Hack Day. - -Again, we don't know -where the next great idea will come from. -We need all the help we can get. -We need a large, diverse Go community. - -* Thank You - -I consider the many people -releasing software for download using “go get,” -sharing their insights via blog posts, -or helping others on the mailing lists or IRC -to be part of this broad open source effort, -part of the Go community. -Everyone here today is also part of that community. - -Thank you in advance -to the presenters -who over the next few days -will take time to share their experiences -using and extending Go. - -Thank you in advance -to all of you in the audience -for taking the time to be here, -to ask questions, -and to let us know -how Go is working for you. -When you go back home, -please continue to share what you've learned. -Even if you don't use Go -for daily work, -we'd love to see what's working for Go -adopted in other contexts, -just as we're always looking for good ideas -to bring back into Go. - -Thank you all again -for making the effort to be here -and for being part of the Go community. - -For the next few days, please: -tell us what we're doing right, -tell us what we're doing wrong, -and help us all work together -to make Go even better. - -Remember to -be friendly, -patient, -welcoming, -considerate, -and respectful. - -Above all, enjoy the conference. diff --git a/content/organizing-go-code.article b/content/organizing-go-code.article deleted file mode 100644 index 17e88313..00000000 --- a/content/organizing-go-code.article +++ /dev/null @@ -1,53 +0,0 @@ -Organizing Go code -16 Aug 2012 -Tags: godoc, gopath, interface, libraries, tools, technical - -Andrew Gerrand - -* Introduction - -Go code is organized differently to that of other languages. This post discusses how to name and package the elements of your Go program to best serve its users. - -* Choose good names - -The names you choose affect how you think about your code, so take care when naming your package and its exported identifiers. - -A package's name provides context for its contents. For instance, the [[http://golang.org/pkg/bytes/][bytes package]] from the standard library exports the `Buffer` type. On its own, the name `Buffer` isn't very descriptive, but when combined with its package name its meaning becomes clear: `bytes.Buffer`. If the package had a less descriptive name, like `util`, the buffer would likely acquire the longer and clumsier name `util.BytesBuffer`. - -Don't be shy about renaming things as you work. As you spend time with your program you will better understand how its pieces fit together and, therefore, what their names should be. There's no need to lock yourself into early decisions. (The [[http://golang.org/cmd/gofmt/][gofmt command]] has a `-r` flag that provides a syntax-aware search and replace, making large-scale refactoring easier.) - -A good name is the most important part of a software interface: the name is the first thing every client of the code will see. A well-chosen name is therefore the starting point for good documentation. Many of the following practices result organically from good naming. - -* Choose a good import path (make your package "go get"-able) - -An import path is the string with which users import a package. It specifies the directory (relative to `$GOROOT/src/pkg` or `$GOPATH/src`) in which the package's source code resides. - -Import paths should be globally unique, so use the path of your source repository as its base. For instance, the `websocket` package from the `go.net` sub-repository has an import path of `"golang.org/x/net/websocket"`. The Go project owns the path `"code.google.com/p/go"`, so that path cannot be used by another author for a different package. Because the repository URL and import path are one and the same, the `go`get` command can fetch and install the package automatically. - -If you don't use a hosted source repository, choose some unique prefix such as a domain, company, or project name. As an example, the import path of all Google's internal Go code starts with the string `"google"`. - -The last element of the import path is typically the same as the package name. For instance, the import path `"net/http"` contains package `http`. This is not a requirement - you can make them different if you like - but you should follow the convention for predictability's sake: a user might be surprised that import `"foo/bar"` introduces the identifier `quux` into the package name space. - -Sometimes people set `GOPATH` to the root of their source repository and put their packages in directories relative to the repository root, such as `"src/my/package"`. On one hand, this keeps the import paths short (`"my/package"` instead of `"github.com/me/project/my/package"`), but on the other it breaks `go`get` and forces users to re-set their `GOPATH` to use the package. Don't do this. - -* Minimize the exported interface - -Your code is likely composed of many small pieces of useful code, and so it is tempting to expose much of that functionality in your package's exported interface. Resist that urge! - -The larger the interface you provide, the more you must support. Users will quickly come to depend on every type, function, variable, and constant you export, creating an implicit contract that you must honor in perpetuity or risk breaking your users' programs. In preparing Go 1 we carefully reviewed the standard library's exported interfaces and removed the parts we weren't ready to commit to. You should take similar care when distributing your own libraries. - -If in doubt, leave it out! - -* What to put into a package - -It is easy to just throw everything into a "grab bag" package, but this dilutes the meaning of the package name (as it must encompass a lot of functionality) and forces the users of small parts of the package to compile and link a lot of unrelated code. - -On the other hand, it is also easy to go overboard in splitting your code into small packages, in which case you will likely becomes bogged down in interface design, rather than just getting the job done. - -Look to the Go standard libraries as a guide. Some of its packages are large and some are small. For instance, the [[http://golang.org/pkg/net/http/][http package]] comprises 17 go source files (excluding tests) and exports 109 identifiers, and the [[http://golang.org/pkg/hash/][hash package]] consists of one file that exports just three declarations. There is no hard and fast rule; both approaches are appropriate given their context. - -With that said, package main is often larger than other packages. Complex commands contain a lot of code that is of little use outside the context of the executable, and often it's simpler to just keep it all in the one place. For instance, the go tool is more than 12000 lines spread across [[http://golang.org/src/cmd/go/][34 files]]. - -* Document your code - -Good documentation is an essential quality of usable and maintainable code. Read the [[http://golang.org/doc/articles/godoc_documenting_go_code.html][Godoc: documenting Go code]] article to learn how to write good doc comments. diff --git a/content/oscon.article b/content/oscon.article deleted file mode 100644 index 4f6854ef..00000000 --- a/content/oscon.article +++ /dev/null @@ -1,35 +0,0 @@ -Go will be at OSCON 2014 -15 Jul 2014 -Tags: conference, oscon - -Francesc Campoy - -* Go will be at OSCON 2014 - -[[http://www.oscon.com][OSCON]], the Open Source Convention, is taking place -from July 20th to the 29th in Portland, Oregon and Go will be central to many -talks. If you are attending make sure you add these to your personal schedule. - -On Monday you'll have the chance to learn Go in these two tutorials: - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34395][Getting Started with Go]] by [[http://twitter.com/spf13][Steve Francia]] (MongoDB) at 9:00am on Monday, 07/21/2014 - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34267][A Quick Introduction to System Tools Programming with Go]] by [[http://twitter.com/macmceniry][Chris McEniry]] (Sony Network Entertainment) at 1:30pm on Monday, 07/21/2014 - -During the rest of the week you can hear how different projects use Go: - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34371][A Recovering Java Developer Learns to Go]] by [[http://twitter.com/mstine][Matt Stine]] (Pivotal) at 1:40pm on Tuesday, 07/22/2014 - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34299][Painless Data Storage with MongoDB and Go]] by [[http://twitter.com/spf13][Steve Francia]] (MongoDB) and [[http://twitter.com/gniemeyer][Gustavo Niemeyer]] (Canonical) at 1:40pm on Tuesday, 07/22/2014 - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/37795][Gophers with Hammers: Fun with Parsing and Generating Go]] by [[http://twitter.com/offbymany][Josh Bleecher Snyder]] (PayPal) at 2:30pm on Tuesday, 07/22/2014 - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34047][Go for Object Oriented Programmers (or OO Programming without Objects)]] by [[http://twitter.com/spf13][Steve Francia]] (MongoDB) at 4:10pm on Wednesday, 07/23/2014 - -- [[http://www.oscon.com/oscon2014/public/schedule/detail/34509][Inside the Go Tour]] by [[http://twitter.com/francesc][Francesc Campoy Flores]] (Google Inc.) at 11:50am on Thursday, 07/24/2014 - -And if you have any questions come to the -[[http://www.oscon.com/oscon2014/public/schedule/detail/37075][Go office hours]] -on Wednesday or come anytime by the Google booth. - -See you at OSCON! diff --git a/content/osconreport.article b/content/osconreport.article deleted file mode 100644 index 3937ade9..00000000 --- a/content/osconreport.article +++ /dev/null @@ -1,73 +0,0 @@ -Go at OSCON -20 Aug 2014 -Tags: conference, oscon - -Francesc Campoy - -* Introduction - -What happens in Portland in July? [[http://www.oscon.com/oscon2014][OSCON]]! At -this year's conference, Go was more present than ever before, with five talks, -two workshops, a -[[http://en.wikipedia.org/wiki/Birds_of_a_feather_(computing)][Birds of a Feather]] -session, and a meetup. - -* Talks - -[[http://twitter.com/mstine][Matt Stine]] talked about his experience switching -from Java to Go with -[[http://www.slideshare.net/mstine/java-devlearnstogooscon][A recovering Java developer learns Go]] -while [[https://twitter.com/spf13][Steve Francia]] presented -[[http://spf13.com/presentation/MongoDB-and-Go][Painless Data Storage with MongoDB and Go]]. -Steve also presented -[[http://spf13.com/presentation/go-for-object-oriented-programmers][Go for Object Oriented Programmers]], -where he explained how some object oriented concepts can be implemented in Go. - -.image osconreport/talks.png _ 800 - -Finally, [[http://twitter.com/offbymany][Josh Bleecher Snyder]] talked about his -experience writing tools to work with Go source code in -[[https://talks.golang.org/2014/hammers.slide#1][Gophers with hammers]], -and [[http://twitter.com/francesc][Francesc Campoy]] talked about all the things -that could have gone wrong and what the Go team did to prevent them -[[http://talks.golang.org/2014/playground.slide][Inside the Go playground]]. - -* Workshops - -At the beginning of OSCON's workshop day, Steve Francia presented how to build a -web application and a CLI tool during -[[http://spf13.com/presentation/first-go-app][Getting started with Go]] to a big -room full of Gophers. - -.image osconreport/workshops.png _ 800 - -In the afternoon, [[https://twitter.com/mmceniry][Chris McEniry]] gave his -[[http://cdn.oreillystatic.com/en/assets/1/event/115/A%20Quick%20Introduction%20to%20System%20Tools%20Programming%20with%20Go%20Presentation.pdf][Quick introduction to system tools programming with Go]] where he went over some useful skills to -write system tools using Go and its standard library. - -* Additional events - -To take advantage of the increased Gopher population in Portland during OSCON, we -organized two extra events: the first [[https://twitter.com/pdxgolang][PDXGolang]] -meetup and a -[[http://www.oscon.com/oscon2014/public/schedule/detail/37775][Birds of a Feather session]]. - -.image osconreport/meetup.png _ 800 - -At the meetup Francesc Campoy talked about -[[http://talks.golang.org/2013/bestpractices.slide][Go Best Practices]] and -[[https://twitter.com/kelseyhightower][Kelsey Hightower]] gave a great -introduction to [[https://github.com/GoogleCloudPlatform/kubernetes][Kubernetes]], -a container management system for clusters written in Go by Google. If you live -in Portland, make sure you [[http://meetup.com/pdx-go][join the group]] and come -along to the next meeting. - -The "Birds of a Feather" (or, more aptly, "Gophers of a Feather") was a lot of -fun for everyone involved. We hope to see more of you there next year. - -* In conclusion - -.image osconreport/random.png _ 800 - -Thanks to all the gophers that participated in OSCON. After the successes of -this year we look forward to more Go fun at OSCON 2015. diff --git a/content/osconreport/meetup.png b/content/osconreport/meetup.png deleted file mode 100644 index 7e4bd697..00000000 Binary files a/content/osconreport/meetup.png and /dev/null differ diff --git a/content/osconreport/random.png b/content/osconreport/random.png deleted file mode 100644 index e0253281..00000000 Binary files a/content/osconreport/random.png and /dev/null differ diff --git a/content/osconreport/talks.png b/content/osconreport/talks.png deleted file mode 100644 index 66b2278e..00000000 Binary files a/content/osconreport/talks.png and /dev/null differ diff --git a/content/osconreport/workshops.png b/content/osconreport/workshops.png deleted file mode 100644 index d4586a42..00000000 Binary files a/content/osconreport/workshops.png and /dev/null differ diff --git a/content/package-names.article b/content/package-names.article deleted file mode 100644 index 4ecc0574..00000000 --- a/content/package-names.article +++ /dev/null @@ -1,249 +0,0 @@ -Package names -4 Feb 2015 -Tags: package, names, style - -Sameer Ajmani - -* Introduction - -Go code is organized into packages. -Within a package, code can refer to any identifier (name) defined within, while -clients of the package may only reference the package's exported types, -functions, constants, and variables. -Such references always include the package name as a prefix: `foo.Bar` refers to -the exported name `Bar` in the imported package named `foo`. - -Good package names make code better. -A package's name provides context for its contents, making it easier for clients -to understand what the package is for and how to use it. -The name also helps package maintainers determine what does and does not belong -in the package as it evolves. -Well-named packages make it easier to find the code you need. - -Effective Go provides -[[https://golang.org/doc/effective_go.html#names][guidelines]] for naming -packages, types, functions, and variables. -This article expands on that discussion and surveys names found in the standard -library. -It also discusses bad package names and how to fix them. - -* Package names - -Good package names are short and clear. -They are lower case, with no `under_scores` or `mixedCaps`. -They are often simple nouns, such as: - -- `time` (provides functionality for measuring and displaying time) -- `list` (implements a doubly linked list) -- `http` (provides HTTP client and server implementations) - -The style of names typical of another language might not be idiomatic in a Go -program. -Here are two examples of names that might be good style in other languages but -do not fit well in Go: - -- `computeServiceClient` -- `priority_queue` - -A Go package may export several types and functions. -For example, a `compute` package could export a `Client` type with methods for -using the service as well as functions for partitioning a compute task across -several clients. - -*Abbreviate*judiciously.* -Package names may be abbreviated when the abbreviation is familiar to the -programmer. -Widely-used packages often have compressed names: - -- `strconv` (string conversion) -- `syscall` (system call) -- `fmt` (formatted I/O) - -On the other hand, if abbreviating a package name makes it ambiguous or unclear, -don't do it. - -*Don't*steal*good*names*from*the*user.* -Avoid giving a package a name that is commonly used in client code. -For example, the buffered I/O package is called `bufio`, not `buf`, since `buf` -is a good variable name for a buffer. - -* Naming package contents - -A package name and its contents' names are coupled, since client code uses them -together. -When designing a package, take the client's point of view. - -*Avoid*stutter.* -Since client code uses the package name as a prefix when referring to the -package contents, the names for those contents need not repeat the package name. -The HTTP server provided by the `http` package is called `Server`, not -`HTTPServer`. -Client code refers to this type as `http.Server`, so there is no ambiguity. - -*Simplify*function*names.* -When a function in package pkg returns a value of type `pkg.Pkg` (or -`*pkg.Pkg`), the function name can often omit the type name without confusion: - - start := time.Now() // start is a time.Time - t, err := time.Parse(time.Kitchen, "6:06PM") // t is a time.Time - ctx = context.WithTimeout(ctx, 10*time.Millisecond) // ctx is a context.Context - ip, ok := userip.FromContext(ctx) // ip is a net.IP - -A function named `New` in package `pkg` returns a value of type `pkg.Pkg`. -This is a standard entry point for client code using that type: - - q := list.New() // q is a *list.List - -When a function returns a value of type `pkg.T`, where `T` is not `Pkg`, the -function name may include `T` to make client code easier to understand. -A common situation is a package with multiple New-like functions: - - d, err := time.ParseDuration("10s") // d is a time.Duration - elapsed := time.Since(start) // elapsed is a time.Duration - ticker := time.NewTicker(d) // ticker is a *time.Ticker - timer := time.NewTimer(d) // timer is a *time.Timer - -Types in different packages can have the same name, because from the client's -point of view such names are discriminated by the package name. -For example, the standard library includes several types named `Reader`, -including `jpeg.Reader`, `bufio.Reader`, and `csv.Reader`. -Each package name fits with `Reader` to yield a good type name. - -If you cannot come up with a package name that's a meaningful prefix for the -package's contents, the package abstraction boundary may be wrong. -Write code that uses your package as a client would, and restructure your -packages if the result seems poor. -This approach will yield packages that are easier for clients to understand and -for the package developers to maintain. - -* Package paths - -A Go package has both a name and a path. -The package name is specified in the package statement of its source files; -client code uses it as the prefix for the package's exported names. -Client code uses the package path when importing the package. -By convention, the last element of the package path is the package name: - - import ( - "fmt" // package fmt - "os/exec" // package exec - "golang.org/x/net/context" // package context - ) - -Build tools map package paths onto directories. -The go tool uses the [[https://golang.org/doc/code.html#GOPATH][GOPATH]] -environment variable to find the source files for path `"github.com/user/hello"` -in directory `$GOPATH/src/github.com/user/hello`. -(This situation should be familiar, of course, but it's important to be clear -about the terminology and structure of packages.) - -*Directories.* -The standard library uses like directories `crypto`, `container`, `encoding`, -and `image` to group packages for related protocols and algorithms. -There is no actual relationship among the packages in one of these directories; -a directory just provides a way to arrange the files. -Any package can import any other package provided the import does not create a -cycle. - -Just as types in different packages can have the same name without ambiguity, -packages in different directories can have the same name. -For example, -[[https://golang.org/pkg/runtime/pprof][runtime/pprof]] provides profiling data -in the format expected by the [[https://code.google.com/p/gperftools][pprof]] -profiling tool, while [[https://golang.org/pkg/net/http/pprof][net/http/pprof]] -provides HTTP endpoints to present profiling data in this format. -Client code uses the package path to import the package, so there is no -confusion. -If a source file needs to import both `pprof` packages, it can -[[https://golang.org/ref/spec#Import_declarations][rename]] one or both locally. -When renaming an imported package, the local name should follow the same -guidelines as package names (lower case, no `under_scores` or `mixedCaps`). - -* Bad package names - -Bad package names make code harder to navigate and maintain. -Here are some guidelines for recognizing and fixing bad names. - -*Avoid*meaningless*package*names.* -Packages named `util`, `common`, or `misc` provide clients with no sense of what -the package contains. -This makes it harder for clients to use the package and makes it harder for -maintainers to keep the package focused. -Over time, they accumulate dependencies that can make compilation significantly -and unnecessarily slower, especially in large programs. -And since such package names are generic, they are more likely to collide with -other packages imported by client code, forcing clients to invent names to -distinguish them. - -*Break*up*generic*packages.* -To fix such packages, look for types and functions with common name elements and -pull them into their own package. -For example, if you have - - package util - func NewStringSet(...string) map[string]bool {...} - func SortStringSet(map[string]bool) []string {...} - -then client code looks like - - set := util.NewStringSet("c", "a", "b") - fmt.Println(util.SortStringSet(set)) - -Pull these functions out of `util` into a new package, choosing a name that fits -the contents: - - package stringset - func New(...string) map[string]bool {...} - func Sort(map[string]bool) []string {...} - -then the client code becomes - - set := stringset.New("c", "a", "b") - fmt.Println(stringset.Sort(set)) - -Once you've made this change, its easier to see how to improve the new package: - - package stringset - type Set map[string]bool - func New(...string) Set {...} - func (s Set) Sort() []string {...} - -which yields even simpler client code: - - set := stringset.New("c", "a", "b") - fmt.Println(set.Sort()) - -The name of the package is a critical piece of its design. -Work to eliminate meaningless package names from your projects. - -*Don't*use*a*single*package*for*all*your*APIs.* -Many well-intentioned programmers put all the interfaces exposed by their -program into a single package named `api`, `types`, or `interfaces`, thinking it -makes it easier to find the entry points to their code base. -This is a mistake. -Such packages suffer from the same problems as those named `util` or `common`, -growing without bound, providing no guidance to users, accumulating -dependencies, and colliding with other imports. -Break them up, perhaps using directories to separate public packages from -implementation. - -*Avoid*unnecessary*package*name*collisions.* -While packages in different directories may have the same name, packages that -are frequently used together should have distinct names. -This reduces confusion and the need for local renaming in client code. -For the same reason, avoid using the same name as popular standard packages like -`io` or `http`. - -* Conclusion - -Package names are central to good naming in Go programs. -Take the time to choose good package names and organize your code well. -This helps clients understand and use your packages and helps maintainers to -grow them gracefully. - -* Further reading - -- [[https://golang.org/doc/effective_go.html][Effective Go]] -- [[https://golang.org/doc/code.html][How to Write Go Code]] -- [[https://blog.golang.org/organizing-go-code][Organizing Go Code (2012 blog post)]] -- [[https://talks.golang.org/2014/organizeio.slide][Organizing Go Code (2014 Google I/O talk)]] diff --git a/content/pipelines.article b/content/pipelines.article deleted file mode 100644 index 05772c94..00000000 --- a/content/pipelines.article +++ /dev/null @@ -1,312 +0,0 @@ -Go Concurrency Patterns: Pipelines and cancellation -13 Mar 2014 -Tags: concurrency, pipelines, cancellation - -Sameer Ajmani - -* Introduction - -Go's concurrency primitives make it easy to construct streaming data pipelines -that make efficient use of I/O and multiple CPUs. This article presents -examples of such pipelines, highlights subtleties that arise when operations -fail, and introduces techniques for dealing with failures cleanly. - -* What is a pipeline? - -There's no formal definition of a pipeline in Go; it's just one of many kinds of -concurrent programs. Informally, a pipeline is a series of _stages_ connected -by channels, where each stage is a group of goroutines running the same -function. In each stage, the goroutines - -- receive values from _upstream_ via _inbound_ channels -- perform some function on that data, usually producing new values -- send values _downstream_ via _outbound_ channels - -Each stage has any number of inbound and outbound channels, except the -first and last stages, which have only outbound or inbound channels, -respectively. The first stage is sometimes called the _source_ or -_producer_; the last stage, the _sink_ or _consumer_. - -We'll begin with a simple example pipeline to explain the ideas and techniques. -Later, we'll present a more realistic example. - -* Squaring numbers - -Consider a pipeline with three stages. - -The first stage, `gen`, is a function that converts a list of integers to a -channel that emits the integers in the list. The `gen` function starts a -goroutine that sends the integers on the channel and closes the channel when all -the values have been sent: - -.code pipelines/square.go /func gen/,/^}/ - -The second stage, `sq`, receives integers from a channel and returns a -channel that emits the square of each received integer. After the -inbound channel is closed and this stage has sent all the values -downstream, it closes the outbound channel: - -.code pipelines/square.go /func sq/,/^}/ - -The `main` function sets up the pipeline and runs the final stage: it receives -values from the second stage and prints each one, until the channel is closed: - -.code pipelines/square.go /func main/,/^}/ - -Since `sq` has the same type for its inbound and outbound channels, we -can compose it any number of times. We can also rewrite `main` as a -range loop, like the other stages: - -.code pipelines/square2.go /func main/,/^}/ - -* Fan-out, fan-in - -Multiple functions can read from the same channel until that channel is closed; -this is called _fan-out_. This provides a way to distribute work amongst a group -of workers to parallelize CPU use and I/O. - -A function can read from multiple inputs and proceed until all are closed by -multiplexing the input channels onto a single channel that's closed when all the -inputs are closed. This is called _fan-in_. - -We can change our pipeline to run two instances of `sq`, each reading from the -same input channel. We introduce a new function, _merge_, to fan in the -results: - -.code pipelines/sqfan.go /func main/,/^}/ - -The `merge` function converts a list of channels to a single channel by starting -a goroutine for each inbound channel that copies the values to the sole outbound -channel. Once all the `output` goroutines have been started, `merge` starts one -more goroutine to close the outbound channel after all sends on that channel are -done. - -Sends on a closed channel panic, so it's important to ensure all sends -are done before calling close. The -[[http://golang.org/pkg/sync/#WaitGroup][`sync.WaitGroup`]] type -provides a simple way to arrange this synchronization: - -.code pipelines/sqfan.go /func merge/,/^}/ - -* Stopping short - -There is a pattern to our pipeline functions: - -- stages close their outbound channels when all the send operations are done. -- stages keep receiving values from inbound channels until those channels are closed. - -This pattern allows each receiving stage to be written as a `range` loop and -ensures that all goroutines exit once all values have been successfully sent -downstream. - -But in real pipelines, stages don't always receive all the inbound -values. Sometimes this is by design: the receiver may only need a -subset of values to make progress. More often, a stage exits early -because an inbound value represents an error in an earlier stage. In -either case the receiver should not have to wait for the remaining -values to arrive, and we want earlier stages to stop producing values -that later stages don't need. - -In our example pipeline, if a stage fails to consume all the inbound values, the -goroutines attempting to send those values will block indefinitely: - -.code pipelines/sqleak.go /first value/,/^}/ - -This is a resource leak: goroutines consume memory and runtime resources, and -heap references in goroutine stacks keep data from being garbage collected. -Goroutines are not garbage collected; they must exit on their own. - -We need to arrange for the upstream stages of our pipeline to exit even when the -downstream stages fail to receive all the inbound values. One way to do this is -to change the outbound channels to have a buffer. A buffer can hold a fixed -number of values; send operations complete immediately if there's room in the -buffer: - - c := make(chan int, 2) // buffer size 2 - c <- 1 // succeeds immediately - c <- 2 // succeeds immediately - c <- 3 // blocks until another goroutine does <-c and receives 1 - -When the number of values to be sent is known at channel creation time, a buffer -can simplify the code. For example, we can rewrite `gen` to copy the list of -integers into a buffered channel and avoid creating a new goroutine: - -.code pipelines/sqbuffer.go /func gen/,/^}/ - -Returning to the blocked goroutines in our pipeline, we might consider adding a -buffer to the outbound channel returned by `merge`: - -.code pipelines/sqbuffer.go /func merge/,/unchanged/ - -While this fixes the blocked goroutine in this program, this is bad code. The -choice of buffer size of 1 here depends on knowing the number of values `merge` -will receive and the number of values downstream stages will consume. This is -fragile: if we pass an additional value to `gen`, or if the downstream stage -reads any fewer values, we will again have blocked goroutines. - -Instead, we need to provide a way for downstream stages to indicate to the -senders that they will stop accepting input. - -* Explicit cancellation - -When `main` decides to exit without receiving all the values from -`out`, it must tell the goroutines in the upstream stages to abandon -the values they're trying it send. It does so by sending values on a -channel called `done`. It sends two values since there are -potentially two blocked senders: - -.code pipelines/sqdone1.go /func main/,/^}/ - -The sending goroutines replace their send operation with a `select` statement -that proceeds either when the send on `out` happens or when they receive a value -from `done`. The value type of `done` is the empty struct because the value -doesn't matter: it is the receive event that indicates the send on `out` should -be abandoned. The `output` goroutines continue looping on their inbound -channel, `c`, so the upstream stages are not blocked. (We'll discuss in a moment -how to allow this loop to return early.) - -.code pipelines/sqdone1.go /func merge/,/unchanged/ - -This approach has a problem: _each_ downstream receiver needs to know the number -of potentially blocked upstream senders and arrange to signal those senders on -early return. Keeping track of these counts is tedious and error-prone. - -We need a way to tell an unknown and unbounded number of goroutines to -stop sending their values downstream. In Go, we can do this by -closing a channel, because -[[http://golang.org/ref/spec#Receive_operator][a receive operation on a closed channel can always proceed immediately, yielding the element type's zero value.]] - -This means that `main` can unblock all the senders simply by closing -the `done` channel. This close is effectively a broadcast signal to -the senders. We extend _each_ of our pipeline functions to accept -`done` as a parameter and arrange for the close to happen via a -`defer` statement, so that all return paths from `main` will signal -the pipeline stages to exit. - -.code pipelines/sqdone3.go /func main/,/^}/ - -Each of our pipeline stages is now free to return as soon as `done` is closed. -The `output` routine in `merge` can return without draining its inbound channel, -since it knows the upstream sender, `sq`, will stop attempting to send when -`done` is closed. `output` ensures `wg.Done` is called on all return paths via -a `defer` statement: - -.code pipelines/sqdone3.go /func merge/,/unchanged/ - -Similarly, `sq` can return as soon as `done` is closed. `sq` ensures its `out` -channel is closed on all return paths via a `defer` statement: - -.code pipelines/sqdone3.go /func sq/,/^}/ - -Here are the guidelines for pipeline construction: - -- stages close their outbound channels when all the send operations are done. -- stages keep receiving values from inbound channels until those channels are closed or the senders are unblocked. - -Pipelines unblock senders either by ensuring there's enough buffer for all the -values that are sent or by explicitly signalling senders when the receiver may -abandon the channel. - -* Digesting a tree - -Let's consider a more realistic pipeline. - -MD5 is a message-digest algorithm that's useful as a file checksum. The command -line utility `md5sum` prints digest values for a list of files. - - % md5sum *.go - d47c2bbc28298ca9befdfbc5d3aa4e65 bounded.go - ee869afd31f83cbb2d10ee81b2b831dc parallel.go - b88175e65fdcbc01ac08aaf1fd9b5e96 serial.go - -Our example program is like `md5sum` but instead takes a single directory as an -argument and prints the digest values for each regular file under that -directory, sorted by path name. - - % go run serial.go . - d47c2bbc28298ca9befdfbc5d3aa4e65 bounded.go - ee869afd31f83cbb2d10ee81b2b831dc parallel.go - b88175e65fdcbc01ac08aaf1fd9b5e96 serial.go - -The main function of our program invokes a helper function `MD5All`, which -returns a map from path name to digest value, then sorts and prints the results: - -.code pipelines/serial.go /func main/,/^}/ - -The `MD5All` function is the focus of our discussion. In -[[pipelines/serial.go][serial.go]], the implementation uses no concurrency and -simply reads and sums each file as it walks the tree. - -.code pipelines/serial.go /MD5All/,/^}/ - -* Parallel digestion - -In [[pipelines/parallel.go][parallel.go]], we split `MD5All` into a two-stage -pipeline. The first stage, `sumFiles`, walks the tree, digests each file in -a new goroutine, and sends the results on a channel with value type `result`: - -.code pipelines/parallel.go /type result/,/}/ HLresult - -`sumFiles` returns two channels: one for the `results` and another for the error -returned by `filepath.Walk`. The walk function starts a new goroutine to -process each regular file, then checks `done`. If `done` is closed, the walk -stops immediately: - -.code pipelines/parallel.go /func sumFiles/,/^}/ - -`MD5All` receives the digest values from `c`. `MD5All` returns early on error, -closing `done` via a `defer`: - -.code pipelines/parallel.go /func MD5All/,/^}/ HLdone - -* Bounded parallelism - -The `MD5All` implementation in [[pipelines/parallel.go][parallel.go]] -starts a new goroutine for each file. In a directory with many large -files, this may allocate more memory than is available on the machine. - -We can limit these allocations by bounding the number of files read in -parallel. In [[pipelines/bounded.go][bounded.go]], we do this by -creating a fixed number of goroutines for reading files. Our pipeline -now has three stages: walk the tree, read and digest the files, and -collect the digests. - -The first stage, `walkFiles`, emits the paths of regular files in the tree: - -.code pipelines/bounded.go /func walkFiles/,/^}/ - -The middle stage starts a fixed number of `digester` goroutines that receive -file names from `paths` and send `results` on channel `c`: - -.code pipelines/bounded.go /func digester/,/^}/ HLpaths - -Unlike our previous examples, `digester` does not close its output channel, as -multiple goroutines are sending on a shared channel. Instead, code in `MD5All` -arranges for the channel to be closed when all the `digesters` are done: - -.code pipelines/bounded.go /fixed number/,/End of pipeline/ HLc - -We could instead have each digester create and return its own output -channel, but then we would need additional goroutines to fan-in the -results. - -The final stage receives all the `results` from `c` then checks the -error from `errc`. This check cannot happen any earlier, since before -this point, `walkFiles` may block sending values downstream: - -.code pipelines/bounded.go /m := make/,/^}/ HLerrc - -* Conclusion - -This article has presented techniques for constructing streaming data pipelines -in Go. Dealing with failures in such pipelines is tricky, since each stage in -the pipeline may block attempting to send values downstream, and the downstream -stages may no longer care about the incoming data. We showed how closing a -channel can broadcast a "done" signal to all the goroutines started by a -pipeline and defined guidelines for constructing pipelines correctly. - -Further reading: - -- [[http://talks.golang.org/2012/concurrency.slide#1][Go Concurrency Patterns]] ([[https://www.youtube.com/watch?v=f6kdp27TYZs][video]]) presents the basics of Go's concurrency primitives and several ways to apply them. -- [[http://blog.golang.org/advanced-go-concurrency-patterns][Advanced Go Concurrency Patterns]] ([[http://www.youtube.com/watch?v=QDDwwePbDtw][video]]) covers more complex uses of Go's primitives, especially `select`. -- Douglas McIlroy's paper [[http://swtch.com/~rsc/thread/squint.pdf][Squinting at Power Series]] shows how Go-like concurrency provides elegant support for complex calculations. diff --git a/content/pipelines/bounded.go b/content/pipelines/bounded.go deleted file mode 100644 index 3977befb..00000000 --- a/content/pipelines/bounded.go +++ /dev/null @@ -1,123 +0,0 @@ -// +build OMIT - -package main - -import ( - "crypto/md5" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "sync" -) - -// walkFiles starts a goroutine to walk the directory tree at root and send the -// path of each regular file on the string channel. It sends the result of the -// walk on the error channel. If done is closed, walkFiles abandons its work. -func walkFiles(done <-chan struct{}, root string) (<-chan string, <-chan error) { - paths := make(chan string) - errc := make(chan error, 1) - go func() { // HL - // Close the paths channel after Walk returns. - defer close(paths) // HL - // No select needed for this send, since errc is buffered. - errc <- filepath.Walk(root, func(path string, info os.FileInfo, err error) error { // HL - if err != nil { - return err - } - if !info.Mode().IsRegular() { - return nil - } - select { - case paths <- path: // HL - case <-done: // HL - return errors.New("walk canceled") - } - return nil - }) - }() - return paths, errc -} - -// A result is the product of reading and summing a file using MD5. -type result struct { - path string - sum [md5.Size]byte - err error -} - -// digester reads path names from paths and sends digests of the corresponding -// files on c until either paths or done is closed. -func digester(done <-chan struct{}, paths <-chan string, c chan<- result) { - for path := range paths { // HLpaths - data, err := ioutil.ReadFile(path) - select { - case c <- result{path, md5.Sum(data), err}: - case <-done: - return - } - } -} - -// MD5All reads all the files in the file tree rooted at root and returns a map -// from file path to the MD5 sum of the file's contents. If the directory walk -// fails or any read operation fails, MD5All returns an error. In that case, -// MD5All does not wait for inflight read operations to complete. -func MD5All(root string) (map[string][md5.Size]byte, error) { - // MD5All closes the done channel when it returns; it may do so before - // receiving all the values from c and errc. - done := make(chan struct{}) - defer close(done) - - paths, errc := walkFiles(done, root) - - // Start a fixed number of goroutines to read and digest files. - c := make(chan result) // HLc - var wg sync.WaitGroup - const numDigesters = 20 - wg.Add(numDigesters) - for i := 0; i < numDigesters; i++ { - go func() { - digester(done, paths, c) // HLc - wg.Done() - }() - } - go func() { - wg.Wait() - close(c) // HLc - }() - // End of pipeline. OMIT - - m := make(map[string][md5.Size]byte) - for r := range c { - if r.err != nil { - return nil, r.err - } - m[r.path] = r.sum - } - // Check whether the Walk failed. - if err := <-errc; err != nil { // HLerrc - return nil, err - } - return m, nil -} - -func main() { - // Calculate the MD5 sum of all files under the specified directory, - // then print the results sorted by path name. - m, err := MD5All(os.Args[1]) - if err != nil { - fmt.Println(err) - return - } - var paths []string - for path := range m { - paths = append(paths, path) - } - sort.Strings(paths) - for _, path := range paths { - fmt.Printf("%x %s\n", m[path], path) - } -} diff --git a/content/pipelines/parallel.go b/content/pipelines/parallel.go deleted file mode 100644 index 5ca3e76a..00000000 --- a/content/pipelines/parallel.go +++ /dev/null @@ -1,111 +0,0 @@ -// +build OMIT - -package main - -import ( - "crypto/md5" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "sync" -) - -// A result is the product of reading and summing a file using MD5. -type result struct { - path string - sum [md5.Size]byte - err error -} - -// sumFiles starts goroutines to walk the directory tree at root and digest each -// regular file. These goroutines send the results of the digests on the result -// channel and send the result of the walk on the error channel. If done is -// closed, sumFiles abandons its work. -func sumFiles(done <-chan struct{}, root string) (<-chan result, <-chan error) { - // For each regular file, start a goroutine that sums the file and sends - // the result on c. Send the result of the walk on errc. - c := make(chan result) - errc := make(chan error, 1) - go func() { // HL - var wg sync.WaitGroup - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.Mode().IsRegular() { - return nil - } - wg.Add(1) - go func() { // HL - data, err := ioutil.ReadFile(path) - select { - case c <- result{path, md5.Sum(data), err}: // HL - case <-done: // HL - } - wg.Done() - }() - // Abort the walk if done is closed. - select { - case <-done: // HL - return errors.New("walk canceled") - default: - return nil - } - }) - // Walk has returned, so all calls to wg.Add are done. Start a - // goroutine to close c once all the sends are done. - go func() { // HL - wg.Wait() - close(c) // HL - }() - // No select needed here, since errc is buffered. - errc <- err // HL - }() - return c, errc -} - -// MD5All reads all the files in the file tree rooted at root and returns a map -// from file path to the MD5 sum of the file's contents. If the directory walk -// fails or any read operation fails, MD5All returns an error. In that case, -// MD5All does not wait for inflight read operations to complete. -func MD5All(root string) (map[string][md5.Size]byte, error) { - // MD5All closes the done channel when it returns; it may do so before - // receiving all the values from c and errc. - done := make(chan struct{}) // HLdone - defer close(done) // HLdone - - c, errc := sumFiles(done, root) // HLdone - - m := make(map[string][md5.Size]byte) - for r := range c { // HLrange - if r.err != nil { - return nil, r.err - } - m[r.path] = r.sum - } - if err := <-errc; err != nil { - return nil, err - } - return m, nil -} - -func main() { - // Calculate the MD5 sum of all files under the specified directory, - // then print the results sorted by path name. - m, err := MD5All(os.Args[1]) - if err != nil { - fmt.Println(err) - return - } - var paths []string - for path := range m { - paths = append(paths, path) - } - sort.Strings(paths) - for _, path := range paths { - fmt.Printf("%x %s\n", m[path], path) - } -} diff --git a/content/pipelines/serial.go b/content/pipelines/serial.go deleted file mode 100644 index 024ef47e..00000000 --- a/content/pipelines/serial.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build OMIT - -package main - -import ( - "crypto/md5" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" -) - -// MD5All reads all the files in the file tree rooted at root and returns a map -// from file path to the MD5 sum of the file's contents. If the directory walk -// fails or any read operation fails, MD5All returns an error. -func MD5All(root string) (map[string][md5.Size]byte, error) { - m := make(map[string][md5.Size]byte) - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { // HL - if err != nil { - return err - } - if !info.Mode().IsRegular() { - return nil - } - data, err := ioutil.ReadFile(path) // HL - if err != nil { - return err - } - m[path] = md5.Sum(data) // HL - return nil - }) - if err != nil { - return nil, err - } - return m, nil -} - -func main() { - // Calculate the MD5 sum of all files under the specified directory, - // then print the results sorted by path name. - m, err := MD5All(os.Args[1]) // HL - if err != nil { - fmt.Println(err) - return - } - var paths []string - for path := range m { - paths = append(paths, path) - } - sort.Strings(paths) // HL - for _, path := range paths { - fmt.Printf("%x %s\n", m[path], path) - } -} diff --git a/content/pipelines/sqbuffer.go b/content/pipelines/sqbuffer.go deleted file mode 100644 index 386790c0..00000000 --- a/content/pipelines/sqbuffer.go +++ /dev/null @@ -1,75 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int, len(nums)) - for _, n := range nums { - out <- n - } - close(out) - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent. -func merge(cs ...<-chan int) <-chan int { - var wg sync.WaitGroup - out := make(chan int, 1) // enough space for the unread inputs - // ... the rest is unchanged ... - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c is closed, then calls wg.Done. - output := func(c <-chan int) { - for n := range c { - out <- n - } - wg.Done() - } - wg.Add(len(cs)) - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() - close(out) - }() - return out -} - -func main() { - in := gen(2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(in) - c2 := sq(in) - - // Consume the first value from output. - out := merge(c1, c2) - fmt.Println(<-out) // 4 or 9 - return - // The second value is sent into out's buffer, and all goroutines exit. -} diff --git a/content/pipelines/sqdone1.go b/content/pipelines/sqdone1.go deleted file mode 100644 index e77818a7..00000000 --- a/content/pipelines/sqdone1.go +++ /dev/null @@ -1,83 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int, len(nums)) - for _, n := range nums { - out <- n - } - close(out) - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent. -func merge(done <-chan struct{}, cs ...<-chan int) <-chan int { - var wg sync.WaitGroup - out := make(chan int) - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c is closed or it receives a value - // from done, then output calls wg.Done. - output := func(c <-chan int) { - for n := range c { - select { - case out <- n: - case <-done: // HL - } - } - wg.Done() - } - // ... the rest is unchanged ... - - wg.Add(len(cs)) - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() - close(out) - }() - return out -} - -func main() { - in := gen(2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(in) - c2 := sq(in) - - // Consume the first value from output. - done := make(chan struct{}, 2) // HL - out := merge(done, c1, c2) - fmt.Println(<-out) // 4 or 9 - - // Tell the remaining senders we're leaving. - done <- struct{}{} // HL - done <- struct{}{} // HL -} diff --git a/content/pipelines/sqdone2.go b/content/pipelines/sqdone2.go deleted file mode 100644 index c6712c8c..00000000 --- a/content/pipelines/sqdone2.go +++ /dev/null @@ -1,80 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int, len(nums)) - for _, n := range nums { - out <- n - } - close(out) - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent. -func merge(done <-chan struct{}, cs ...<-chan int) <-chan int { - var wg sync.WaitGroup - out := make(chan int) - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c or done is closed, then calls - // wg.Done. - output := func(c <-chan int) { - for n := range c { - select { - case out <- n: - case <-done: // HL - } - } - wg.Done() - } - wg.Add(len(cs)) - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() - close(out) - }() - return out -} - -func main() { - in := gen(2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(in) - c2 := sq(in) - - // Consume the first value from output. - done := make(chan struct{}) // HL - out := merge(done, c1, c2) // HL - fmt.Println(<-out) // 4 or 9 - - // Tell the remaining senders we're leaving. - close(done) // HL -} diff --git a/content/pipelines/sqdone3.go b/content/pipelines/sqdone3.go deleted file mode 100644 index 5540a42f..00000000 --- a/content/pipelines/sqdone3.go +++ /dev/null @@ -1,92 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(done <-chan struct{}, nums ...int) <-chan int { - out := make(chan int, len(nums)) - for _, n := range nums { - // We ignore done here because these sends cannot block. - out <- n - } - close(out) - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in or done is closed. Then sq closes the returned channel. -func sq(done <-chan struct{}, in <-chan int) <-chan int { - out := make(chan int) - go func() { - defer close(out) // HL - for n := range in { - select { - case out <- n * n: - case <-done: - return // HL - } - } - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent or after done is closed. -func merge(done <-chan struct{}, cs ...<-chan int) <-chan int { - var wg sync.WaitGroup - out := make(chan int) - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c or done is closed, then calls - // wg.Done. - output := func(c <-chan int) { - defer wg.Done() // HL - for n := range c { - select { - case out <- n: - case <-done: - return // HL - } - } - } - // ... the rest is unchanged ... - - wg.Add(len(cs)) - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() - close(out) - }() - return out -} - -func main() { - // Set up a done channel that's shared by the whole pipeline, - // and close that channel when this pipeline exits, as a signal - // for all the goroutines we started to exit. - done := make(chan struct{}) // HL - defer close(done) // HL - - in := gen(done, 2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(done, in) - c2 := sq(done, in) - - // Consume the first value from output. - out := merge(done, c1, c2) - fmt.Println(<-out) // 4 or 9 - - // done will be closed by the deferred call. // HL -} diff --git a/content/pipelines/sqfan.go b/content/pipelines/sqfan.go deleted file mode 100644 index be5138c1..00000000 --- a/content/pipelines/sqfan.go +++ /dev/null @@ -1,75 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int) - go func() { - for _, n := range nums { - out <- n - } - close(out) - }() - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent. -func merge(cs ...<-chan int) <-chan int { - var wg sync.WaitGroup // HL - out := make(chan int) - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c is closed, then calls wg.Done. - output := func(c <-chan int) { - for n := range c { - out <- n - } - wg.Done() // HL - } - wg.Add(len(cs)) // HL - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() // HL - close(out) - }() - return out -} - -func main() { - in := gen(2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(in) - c2 := sq(in) - - // Consume the merged output from c1 and c2. - for n := range merge(c1, c2) { - fmt.Println(n) // 4 then 9, or 9 then 4 - } -} diff --git a/content/pipelines/sqleak.go b/content/pipelines/sqleak.go deleted file mode 100644 index 43332054..00000000 --- a/content/pipelines/sqleak.go +++ /dev/null @@ -1,77 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "sync" -) - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int) - go func() { - for _, n := range nums { - out <- n - } - close(out) - }() - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -// merge receives values from each input channel and sends them on the returned -// channel. merge closes the returned channel after all the input values have -// been sent. -func merge(cs ...<-chan int) <-chan int { - var wg sync.WaitGroup - out := make(chan int) - - // Start an output goroutine for each input channel in cs. output - // copies values from c to out until c is closed, then calls wg.Done. - output := func(c <-chan int) { - for n := range c { - out <- n - } - wg.Done() - } - wg.Add(len(cs)) - for _, c := range cs { - go output(c) - } - - // Start a goroutine to close out once all the output goroutines are - // done. This must start after the wg.Add call. - go func() { - wg.Wait() - close(out) - }() - return out -} - -func main() { - in := gen(2, 3) - - // Distribute the sq work across two goroutines that both read from in. - c1 := sq(in) - c2 := sq(in) - - // Consume the first value from output. - out := merge(c1, c2) - fmt.Println(<-out) // 4 or 9 - return - // Since we didn't receive the second value from out, - // one of the output goroutines is hung attempting to send it. -} diff --git a/content/pipelines/square.go b/content/pipelines/square.go deleted file mode 100644 index e558bae2..00000000 --- a/content/pipelines/square.go +++ /dev/null @@ -1,40 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int) - go func() { - for _, n := range nums { - out <- n - } - close(out) - }() - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -func main() { - // Set up the pipeline. - c := gen(2, 3) - out := sq(c) - - // Consume the output. - fmt.Println(<-out) // 4 - fmt.Println(<-out) // 9 -} diff --git a/content/pipelines/square2.go b/content/pipelines/square2.go deleted file mode 100644 index a4108ee6..00000000 --- a/content/pipelines/square2.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build OMIT - -package main - -import "fmt" - -// gen sends the values in nums on the returned channel, then closes it. -func gen(nums ...int) <-chan int { - out := make(chan int) - go func() { - for _, n := range nums { - out <- n - } - close(out) - }() - return out -} - -// sq receives values from in, squares them, and sends them on the returned -// channel, until in is closed. Then sq closes the returned channel. -func sq(in <-chan int) <-chan int { - out := make(chan int) - go func() { - for n := range in { - out <- n * n - } - close(out) - }() - return out -} - -func main() { - // Set up the pipeline and consume the output. - for n := range sq(sq(gen(2, 3))) { - fmt.Println(n) // 16 then 81 - } -} diff --git a/content/playground.article b/content/playground.article deleted file mode 100644 index d82ce19b..00000000 --- a/content/playground.article +++ /dev/null @@ -1,363 +0,0 @@ -Inside the Go Playground -12 Dec 2013 -Tags: playground - -Andrew Gerrand - -* Introduction - -In September 2010 we [[http://blog.golang.org/introducing-go-playground][introduced the Go Playground]], -a web service that compiles and executes arbitrary Go code and returns the -program output. - -If you're a Go programmer then you have probably already used the playground -by using the [[http://play.golang.org][Go Playground]] directly, -taking the [[http://tour.golang.org][Go Tour]], -or running [[http://golang.org/pkg/strings/#pkg-examples][executable examples]] -from the Go documentation. - -You may also have used it by clicking one of the "Run" buttons in a slide -deck on [[http://talks.golang.org/][talks.golang.org]] or a post on this -very blog -(such as the [[http://blog.golang.org/strings][recent article on Strings]]). - -In this article we will take a look at how the playground is implemented -and integrated with these services. -The implementation involves a variant operating system environment and runtime -and our description here assumes you have some familiarity with systems -programming using Go. - - -* Overview - -.image playground/overview.png - -The playground service has three parts: - -- A back end that runs on Google's servers. It receives RPC requests, compiles the user program using the gc tool chain, executes the user program, and returns the program output (or compilation errors) as the RPC response. -- A front end that runs on [[https://developers.google.com/appengine/][Google App Engine]]. It receives HTTP requests from the client and makes corresponding RPC requests to the back end. It also does some caching. -- A JavaScript client that implements the user interface and makes HTTP requests to the front end. - - -* The back end - -The back end program itself is trivial, so we won't discuss its implementation -here. The interesting part is how we safely execute arbitrary user code in a -secure environment while still providing core functionality such as time, the -network, and the file system. - -To isolate user programs from Google's infrastructure, the back end runs -them under [[https://developers.google.com/native-client/][Native Client]] -(or "NaCl"), a technology developed by Google to permit the safe execution of -x86 programs inside web browsers. The back end uses a special version of the gc -tool chain that generates NaCl executables. - -(This special tool chain will be merged into the core for Go 1.3. -To learn more, read the [[http://golang.org/s/go13nacl][design document]]. -If you want to play with NaCl before then, you can -[[https://code.google.com/r/rsc-go13nacl/source/checkout][check out a fork]] -that has all the changes.) - -NaCl limits the amount of CPU and RAM a program may consume, and it prevents -programs from accessing the network or file system. -This presents a problem, however. -Go's concurrency and networking support are among its key strengths, -and access to the file system is vital for many programs. -To demonstrate concurrency effectively we need time, and to demonstrate -networking and the file system we obviously need a network and a file system. - -Although all these things are supported today, the first version of the -playground, launched in 2010, had none of them. -The current time was fixed at 10 November 2009, `time.Sleep` had no effect, -and most functions of the `os` and `net` packages were stubbed out to -return an `EINVALID` error. - -A year ago we -[[https://groups.google.com/d/msg/golang-nuts/JBsCrDEVyVE/30MaQsiQcWoJ][implemented fake time]] -in the playground, so that programs that sleep would behave correctly. -A more recent update to the playground introduced a fake network stack and a -fake file system, making the playground's tool chain similar to a normal -Go tool chain. -These facilities are described in the following sections. - - -** Faking time - -Playground programs are limited in the amount of CPU time and memory they can -use, but they are also restricted in how much real time they can use. -This is because each running program consumes resources on the back end -and any stateful infrastructure between it and the client. -Limiting the run time of each playground program makes our service more -predictable and defends us against denial of service attacks. - -But these restrictions become stifling when running code that uses time. -The [[http://talks.golang.org/2012/concurrency.slide][Go Concurrency Patterns]] -talk demonstrates concurrency with examples that use timing functions like -[[http://golang.org/pkg/time/#Sleep][`time.Sleep`]] and -[[http://golang.org/pkg/time/#After][`time.After`]]. -When run under early versions of the playground, these programs' sleeps would -have no effect and their behavior would be strange (and sometimes wrong). - -By using a clever trick we can make a Go program _think_ that it is sleeping, -when really the sleeps take no time at all. -To explain the trick we first need to understand how the scheduler manages -sleeping goroutines. - -When a goroutine calls `time.Sleep` (or similar) the scheduler adds a timer to -a heap of pending timers and puts the goroutine to sleep. -Meanwhile, a special timer goroutine manages that heap. -When the timer goroutine starts it tells the scheduler to wake -it when the next pending timer is ready to fire and then sleeps. -When it wakes up it checks which timers have expired, wakes the appropriate -goroutines, and goes back to sleep. - -The trick is to change the condition that wakes the timer goroutine. -Instead of waking it after a specific time period, we modify the scheduler to -wait for a deadlock; the state where all goroutines are blocked. - -The playground version of the runtime maintains its own internal clock. When -the modified scheduler detects a deadlock it checks whether any timers are -pending. If so, it advances the internal clock to the trigger time of the -earliest timer and then wakes the timer goroutine. Execution continues and the -program believes that time has passed, when in fact the sleep was nearly -instantaneous. - -These changes to the scheduler can be found in [[https://code.google.com/r/rsc-go13nacl/source/diff?spec=svnc9a5be0fa2db5edcf8f8788da9f7eed323df6c07&r=c9a5be0fa2db5edcf8f8788da9f7eed323df6c07&format=side&path=/src/pkg/runtime/proc.c#sc_svn35d5bae6aac826e6db1851ea14fe9f8da95088a8_2314][`proc.c`]] and [[https://code.google.com/r/rsc-go13nacl/source/diff?spec=svnc9a5be0fa2db5edcf8f8788da9f7eed323df6c07&r=c9a5be0fa2db5edcf8f8788da9f7eed323df6c07&format=side&path=/src/pkg/runtime/time.goc#sc_svnc9a5be0fa2db5edcf8f8788da9f7eed323df6c07_178][`time.goc`]]. - -Fake time fixes the issue of resource exhaustion on the back end, but what -about the program output? It would be odd to see a program that sleeps run to -completion correctly without taking any time. - -The following program prints the current time each second and then exits after -three seconds. Try running it. - -.play -edit playground/time.go /^func main/,$ - -How does this work? It is a collaboration between the back end, front end, and client. - -We capture the timing of each write to standard output and standard error and -provide it to the client. Then the client can "play back" the writes with the -correct timing, so that the output appears just as if the program were running -locally. - -The playground's `runtime` package provides a special -[[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/runtime/sys_nacl_amd64p32.s?r=1f01be1a1dc2#54][`write` function]] -that includes a small "playback header" before each write. -The playback header comprises a magic string, the current time, and the -length of the write data. A write with a playback header has this structure: - - 0 0 P B <8-byte time> <4-byte data length> - -The raw output of the program above looks like this: - - \x00\x00PB\x11\x74\xef\xed\xe6\xb3\x2a\x00\x00\x00\x00\x1e2009-11-10 23:00:01 +0000 UTC - \x00\x00PB\x11\x74\xef\xee\x22\x4d\xf4\x00\x00\x00\x00\x1e2009-11-10 23:00:02 +0000 UTC - \x00\x00PB\x11\x74\xef\xee\x5d\xe8\xbe\x00\x00\x00\x00\x1e2009-11-10 23:00:03 +0000 UTC - -The front end parses this output as a series of events -and returns a list of events to the client as a JSON object: - - { - "Errors": "", - "Events": [ - { - "Delay": 1000000000, - "Message": "2009-11-10 23:00:01 +0000 UTC\n" - }, - { - "Delay": 1000000000, - "Message": "2009-11-10 23:00:02 +0000 UTC\n" - }, - { - "Delay": 1000000000, - "Message": "2009-11-10 23:00:03 +0000 UTC\n" - } - ] - } - -The JavaScript client (running in the user's web browser) then plays back the -events using the provided delay intervals. -To the user it appears that the program is running in real time. - - -** Faking the file system - -Programs built with the Go's NaCl tool chain cannot access the local machine's -file system. Instead, the `syscall` package's file-related functions -(`Open`, `Read`, `Write`, and so on) operate on an in-memory file system -that is implemented by the `syscall` package itself. -Since package `syscall` is the interface between the Go code and the operating -system kernel, user programs see the file system exactly the same way as they -would a real one. - -The following example program writes data to a file, and then copies -its contents to standard output. Try running it. (You can edit it, too!) - -.play -edit playground/os.go /^func main/,$ - -When a process starts, the file system is populated with some devices under -`/dev` and an empty `/tmp` directory. The program can manipulate the file -system as usual, but when the process exits any changes to the file system are -lost. - -There is also a provision to load a zip file into the file system at init time -(see [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/unzip_nacl.go][`unzip_nacl.go`]]). -So far we have only used the unzip facility to provide the data files required -to run the standard library tests, but we intend to provide playground programs -with a set of files that can be used in documentation examples, blog posts, and -the Go Tour. - -The implementation can be found in the -[[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fs_nacl.go][`fs_nacl.go`]] and -[[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fd_nacl.go][`fd_nacl.go`]] files -(which, by virtue of their `_nacl` suffix, are built into package `syscall` only -when `GOOS` is set to `nacl`). - -The file system itself is represented by the -[[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fs_nacl.go?r=5317d308abe4078f68aecc14fd5fb95303d62a06#25][`fsys` struct]], -of which a global instance (named `fs`) is created during init time. -The various file-related functions then operate on `fs` instead of making the -actual system call. -For instance, here is the [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fs_nacl.go?r=1f01be1a1dc2#467][`syscall.Open`]] function: - - func Open(path string, openmode int, perm uint32) (fd int, err error) { - fs.mu.Lock() - defer fs.mu.Unlock() - f, err := fs.open(path, openmode, perm&0777|S_IFREG) - if err != nil { - return -1, err - } - return newFD(f), nil - } - -File descriptors are tracked by a global slice named -[[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fd_nacl.go?r=1f01be1a1dc2#16][`files`]]. -Each file descriptor corresponds to a [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fd_nacl.go?r=1f01be1a1dc2#22][`file`]] -and each `file` provides a value that implements the [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fd_nacl.go?r=1f01be1a1dc2#29][`fileImpl`]] interface. -There are several implementations of the interface: - -- regular files and devices (such as `/dev/random`) are represented by [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fs_nacl.go?r=1f01be1a1dc2#58][`fsysFile`]], -- standard input, output, and error are instances of [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/fd_nacl.go?r=1f01be1a1dc2#209][`naclFile`]], which uses system calls to interact with the actual files (these are a playground program's only way to interact with the outside world), -- network sockets have their own implementation, discussed in the next section. - - -** Faking the network - -Like the file system, the playground's network stack is an in-process fake -implemented by the `syscall` package. It permits playground projects to use -the loopback interface (`127.0.0.1`). Requests to other hosts will fail. - -For an executable example, run the following program. It listens on a TCP port, -waits for an incoming connection, copies the data from that connection to -standard output, and exits. In another goroutine, it makes a connection to the -listening port, writes a string to the connection, and closes it. - -.play -edit playground/net.go /^func main/,$ - -The interface to the network is more complex than the one for files, so the -implementation of the fake network is larger and more complex than the fake -file system. It must simulate read and write timeouts, different address types -and protocols, and so on. - -The implementation can be found in [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/net_nacl.go?r=1f01be1a1dc2][`net_nacl.go`]]. -A good place to start reading is [[https://code.google.com/r/rsc-go13nacl/source/browse/src/pkg/syscall/net_nacl.go?r=1f01be1a1dc2#419][`netFile`]], the network socket implementation of the `fileImpl` interface. - - -* The front end - -The playground front end is another simple program (shorter than 100 lines). -It receives HTTP requests from the client, makes RPC requests to the back end, -and does some caching. - -The front end serves an HTTP handler at `http://golang.org/compile`. -The handler expects a POST request with a `body` field -(the Go program to run) and an optional `version` field -(for most clients this should be `"2"`). - -When the front end receives a compilation request it first checks -[[https://developers.google.com/appengine/docs/memcache/][memcache]] -to see if it has cached the results of a previous compilation of that source. -If found, it returns the cached response. -The cache prevents popular programs such as those on the -[[http://golang.org/][Go home page]] from overloading the back ends. -If there is no cached response, the front end makes an RPC request to the back -end, stores the response in memcache, parses the playback events, and returns -a JSON object to the client as the HTTP response (as described above). - - -* The client - -The various sites that use the playground each share some common JavaScript -code for setting up the user interface (the code and output boxes, the run -button, and so on) and communicating with the playground front end. - -This implementation is in the file -[[https://code.google.com/p/go/source/browse/godoc/static/playground.js?repo=tools][`playground.js`]] -in the `go.tools` repository, which can be imported from the -[[http://godoc.org/code.google.com/p/go.tools/godoc/static][`go.tools/godoc/static`]] package. -Some of it is clean and some is a bit crufty, as it is the result of -consolidating several divergent implementations of the client code. - -The [[https://code.google.com/p/go/source/browse/godoc/static/playground.js?repo=tools#226][`playground`]] -function takes some HTML elements and turns them into an interactive -playground widget. You should use this function if you want to put the -playground on your own site (see 'Other clients' below). - -The [[https://code.google.com/p/go/source/browse/godoc/static/playground.js?repo=tools#6][`Transport`]] -interface (not formally defined, this being JavaScript) -abstracts the user interface from the means of talking to the web front end. -[[https://code.google.com/p/go/source/browse/godoc/static/playground.js?repo=tools#43][`HTTPTransport`]] -is an implementation of `Transport` that speaks the HTTP-based protocol -described earlier. -[[https://code.google.com/p/go/source/browse/godoc/static/playground.js?repo=tools#115][`SocketTransport`]] -is another implementation that speaks WebSocket (see 'Playing offline' below). - -To comply with the [[http://en.wikipedia.org/wiki/Same-origin_policy][same-origin policy]], -the various web servers (godoc, for instance) proxy requests to -`/compile` through to the playground service at `http://golang.org/compile`. -The common [[http://godoc.org/code.google.com/p/go.tools/playground][`go.tools/playground`]] -package does this proxying. - - -* Playing offline - -Both the [[http://tour.golang.org][Go Tour]] and the -[[http://godoc.org/code.google.com/p/go.talks/present][Present Tool]] can be -run offline. This is great for people with limited internet connectivity -or presenters at conferences who cannot (and _should_ not) rely on a working -internet connection. - -To run offline, the tools run their own version of the playground back end on -the local machine. The back end uses a regular Go tool chain with none of the -aforementioned modifications and uses a WebSocket to communicate with the -client. - -The WebSocket back end implementation can be found in the -[[http://godoc.org/code.google.com/p/go.tools/playground/socket][`go.tools/playground/socket`]] package. -The [[http://talks.golang.org/2012/insidepresent.slide#1][Inside Present]] talk discusses this code in detail. - - -* Other clients - -The playground service is used by more than just the official Go project -([[https://gobyexample.com/][Go by Example]] is one other instance) -and we are happy for you to use it on your own site. All we ask is that -you [[mailto:golang-dev@googlegroups.com][contact us first]], -use a unique user agent in your requests (so we can identify you), and that -your service is of benefit to the Go community. - - -* Conclusion - -From godoc to the tour to this very blog, the playground has become an -essential part of our Go documentation story. With the recent additions -of the fake file system and network stack we are excited to expand -our learning materials to cover those areas. - -But, ultimately, the playground is just the tip of the iceberg. -With Native Client support scheduled for Go 1.3, -we look forward to seeing what the community can do with it. - -_This_article_is_part_12_of_the_ -[[http://blog.gopheracademy.com/go-advent-2013][Go Advent Calendar]], -_a_series_of_daily_blog_posts_throughout_December_._ diff --git a/content/playground/net.go b/content/playground/net.go deleted file mode 100644 index eb8b759c..00000000 --- a/content/playground/net.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build OMIT - -package main - -import ( - "io" - "log" - "net" - "os" -) - -func main() { - l, err := net.Listen("tcp", "127.0.0.1:4000") - if err != nil { - log.Fatal(err) - } - defer l.Close() - - go dial() - - c, err := l.Accept() - if err != nil { - log.Fatal(err) - } - defer c.Close() - - io.Copy(os.Stdout, c) -} - -func dial() { - c, err := net.Dial("tcp", "127.0.0.1:4000") - if err != nil { - log.Fatal(err) - } - defer c.Close() - c.Write([]byte("Hello, network\n")) -} diff --git a/content/playground/os.go b/content/playground/os.go deleted file mode 100644 index cf0cd32f..00000000 --- a/content/playground/os.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "io/ioutil" - "log" -) - -func main() { - const filename = "/tmp/file.txt" - - err := ioutil.WriteFile(filename, []byte("Hello, file system\n"), 0644) - if err != nil { - log.Fatal(err) - } - - b, err := ioutil.ReadFile(filename) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("%s", b) -} diff --git a/content/playground/overview.png b/content/playground/overview.png deleted file mode 100644 index 10d5a36e..00000000 Binary files a/content/playground/overview.png and /dev/null differ diff --git a/content/playground/time.go b/content/playground/time.go deleted file mode 100644 index fb1f3149..00000000 --- a/content/playground/time.go +++ /dev/null @@ -1,22 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "time" -) - -func main() { - stop := time.After(3 * time.Second) - tick := time.NewTicker(1 * time.Second) - defer tick.Stop() - for { - select { - case <-tick.C: - fmt.Println(time.Now()) - case <-stop: - return - } - } -} diff --git a/content/preview-of-go-version-1.article b/content/preview-of-go-version-1.article deleted file mode 100644 index bb7d26f0..00000000 --- a/content/preview-of-go-version-1.article +++ /dev/null @@ -1,19 +0,0 @@ -A preview of Go version 1 -5 Oct 2011 -Tags: go1, release - -Russ Cox - -* Introduction - -We want to be able to provide a stable base for people using Go. People should be able to write Go programs and expect that they will continue to compile and run without change, on a timescale of years. Similarly, people should be able to write books about Go, be able to say which version of Go the book is describing, and have that version number still be meaningful much later. None of these properties is true for Go today. - -We propose to issue a Go release early next year that will be called “Go version 1”, Go 1 for short, that will be the first Go release to be stable in this way. Code that compiles in Go version 1 should, with few exceptions, continue to compile throughout the lifetime of that version, as we issue updates and bug fixes such as Go version 1.1, 1.2, and so on. It will also be maintained with fixes for bugs and security flaws even as other versions may evolve. Also, production environments such as Google App Engine will support it for an extended time. - -Go version 1 will be a stable language with stable libraries. Other than critical fixes, changes made to the library and packages for versions 1.1, 1.2 and so on may add functionality but will not break existing Go version 1 programs. - -Our goal is for Go 1 to be a stable version of today’s Go, not a wholesale rethinking of the language. In particular, we are explicitly resisting any efforts to design new language features “by committee.” - -However, there are various changes to the Go language and packages that we have intended for some time and prototyped but have not deployed yet, primarily because they are significant and backwards-incompatible. If Go 1 is to be long-lasting, it is important that we plan, announce, implement, and test these changes as part of the preparation of Go 1, rather than delay them until after it is released and thereby introduce divergence that contradicts our goals. - -Today, we are publishing our preliminary [[https://docs.google.com/document/pub?id=1ny8uI-_BHrDCZv_zNBSthNKAMX_fR_0dc6epA6lztRE][plan for Go 1]] for feedback from the Go community. If you have feedback, please reply to the [[http://groups.google.com/group/golang-nuts/browse_thread/thread/badc4f323431a4f6][thread on the golang-nuts mailing list]]. diff --git a/content/profiling-go-programs.article b/content/profiling-go-programs.article deleted file mode 100644 index 0b74eeeb..00000000 --- a/content/profiling-go-programs.article +++ /dev/null @@ -1,651 +0,0 @@ -Profiling Go Programs -24 Jun 2011 -Tags: benchmark, pprof, profiling, technical - -Russ Cox, July 2011; updated by Shenghou Ma, May 2013 - -* Introduction - -At Scala Days 2011, Robert Hundt presented a paper titled -[[http://research.google.com/pubs/pub37122.html][Loop Recognition in C++/Java/Go/Scala.]] -The paper implemented a specific loop finding algorithm, such as you might use -in a flow analysis pass of a compiler, in C++, Go, Java, Scala, and then used -those programs to draw conclusions about typical performance concerns in these -languages. -The Go program presented in that paper runs quite slowly, making it -an excellent opportunity to demonstrate how to use Go's profiling tools to take -a slow program and make it faster. - -_By_using_Go's_profiling_tools_to_identify_and_correct_specific_bottlenecks,_we_can_make_the_Go_loop_finding_program_run_an_order_of_magnitude_faster_and_use_6x_less_memory._ -(Update: Due to recent optimizations of `libstdc++` in `gcc`, the memory reduction is now 3.7x.) - -Hundt's paper does not specify which versions of the C++, Go, Java, and Scala -tools he used. -In this blog post, we will be using the most recent weekly snapshot of the `6g` -Go compiler and the version of `g++` that ships with the Ubuntu Natty -distribution. -(We will not be using Java or Scala, because we are not skilled at writing efficient -programs in either of those languages, so the comparison would be unfair. -Since C++ was the fastest language in the paper, the comparisons here with C++ should -suffice.) -(Update: In this updated post, we will be using the most recent development snapshot -of the Go compiler on amd64 and the most recent version of `g++` -- 4.8.0, which was -released in March 2013.) - - $ go version - go version devel +08d20469cc20 Tue Mar 26 08:27:18 2013 +0100 linux/amd64 - $ g++ --version - g++ (GCC) 4.8.0 - Copyright (C) 2013 Free Software Foundation, Inc. - ... - $ - -The programs are run on a computer with a 3.4GHz Core i7-2600 CPU and 16 GB of -RAM running Gentoo Linux's 3.8.4-gentoo kernel. -The machine is running with CPU frequency scaling disabled via - - $ sudo bash - # for i in /sys/devices/system/cpu/cpu[0-7] - do - echo performance > $i/cpufreq/scaling_governor - done - # - -We've taken [[http://code.google.com/p/multi-language-bench/][Hundt's benchmark programs]] -in C++ and Go, combined each into a single source file, and removed all but one -line of output. -We'll time the program using Linux's `time` utility with a format that shows user time, -system time, real time, and maximum memory usage: - - $ cat xtime - #!/bin/sh - /usr/bin/time -f '%Uu %Ss %er %MkB %C' "$@" - $ - - $ make havlak1cc - g++ -O3 -o havlak1cc havlak1.cc - $ ./xtime ./havlak1cc - # of loops: 76002 (total 3800100) - loop-0, nest: 0, depth: 0 - 17.70u 0.05s 17.80r 715472kB ./havlak1cc - $ - - $ make havlak1 - go build havlak1.go - $ ./xtime ./havlak1 - # of loops: 76000 (including 1 artificial root node) - 25.05u 0.11s 25.20r 1334032kB ./havlak1 - $ - -The C++ program runs in 17.80 seconds and uses 700 MB of memory. -The Go program runs in 25.20 seconds and uses 1302 MB of memory. -(These measurements are difficult to reconcile with the ones in the paper, but the -point of this post is to explore how to use `go tool pprof`, not to reproduce the -results from the paper.) - -To start tuning the Go program, we have to enable profiling. -If the code used the [[http://golang.org/pkg/testing/][Go testing package]]'s -benchmarking support, we could use gotest's standard `-cpuprofile` and `-memprofile` -flags. -In a standalone program like this one, we have to import `runtime/pprof` and add a few -lines of code: - - var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") - - func main() { - flag.Parse() - if *cpuprofile != "" { - f, err := os.Create(*cpuprofile) - if err != nil { - log.Fatal(err) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - ... - -The new code defines a flag named `cpuprofile`, calls the -[[http://golang.org/pkg/flag/][Go flag library]] to parse the command line flags, -and then, if the `cpuprofile` flag has been set on the command line, -[[http://golang.org/pkg/runtime/pprof/#StartCPUProfile][starts CPU profiling]] -redirected to that file. -The profiler requires a final call to -[[http://golang.org/pkg/runtime/pprof/#StopCPUProfile][`StopCPUProfile`]] to -flush any pending writes to the file before the program exits; we use `defer` -to make sure this happens as `main` returns. - -After adding that code, we can run the program with the new `-cpuprofile` flag -and then run `go tool pprof` to interpret the profile. - - $ make havlak1.prof - ./havlak1 -cpuprofile=havlak1.prof - # of loops: 76000 (including 1 artificial root node) - $ go tool pprof havlak1 havlak1.prof - Welcome to pprof! For help, type 'help'. - (pprof) - -The `go tool pprof` program is a slight variant of -[[https://code.google.com/p/gperftools/wiki/GooglePerformanceTools][Google's `pprof` C++ profiler]]. -The most important command is `topN`, which shows the top `N` samples in the profile: - - (pprof) top10 - Total: 2525 samples - 298 11.8% 11.8% 345 13.7% runtime.mapaccess1_fast64 - 268 10.6% 22.4% 2124 84.1% main.FindLoops - 251 9.9% 32.4% 451 17.9% scanblock - 178 7.0% 39.4% 351 13.9% hash_insert - 131 5.2% 44.6% 158 6.3% sweepspan - 119 4.7% 49.3% 350 13.9% main.DFS - 96 3.8% 53.1% 98 3.9% flushptrbuf - 95 3.8% 56.9% 95 3.8% runtime.aeshash64 - 95 3.8% 60.6% 101 4.0% runtime.settype_flush - 88 3.5% 64.1% 988 39.1% runtime.mallocgc - -When CPU profiling is enabled, the Go program stops about 100 times per second -and records a sample consisting of the program counters on the currently executing -goroutine's stack. -The profile has 2525 samples, so it was running for a bit over 25 seconds. -In the `go tool pprof` output, there is a row for each function that appeared in -a sample. -The first two columns show the number of samples in which the function was running -(as opposed to waiting for a called function to return), as a raw count and as a -percentage of total samples. -The `runtime.mapaccess1_fast64` function was running during 298 samples, or 11.8%. -The `top10` output is sorted by this sample count. -The third column shows the running total during the listing: -the first three rows account for 32.4% of the samples. -The fourth and fifth columns show the number of samples in which the function appeared -(either running or waiting for a called function to return). -The `main.FindLoops` function was running in 10.6% of the samples, but it was on the -call stack (it or functions it called were running) in 84.1% of the samples. - -To sort by the fourth and fifth columns, use the `-cum` (for cumulative) flag: - - (pprof) top5 -cum - Total: 2525 samples - 0 0.0% 0.0% 2144 84.9% gosched0 - 0 0.0% 0.0% 2144 84.9% main.main - 0 0.0% 0.0% 2144 84.9% runtime.main - 0 0.0% 0.0% 2124 84.1% main.FindHavlakLoops - 268 10.6% 10.6% 2124 84.1% main.FindLoops - (pprof) top5 -cum - -In fact the total for `main.FindLoops` and `main.main` should have been 100%, but -each stack sample only includes the bottom 100 stack frames; during about a quarter -of the samples, the recursive `main.DFS` function was more than 100 frames deeper -than `main.main` so the complete trace was truncated. - -The stack trace samples contain more interesting data about function call relationships -than the text listings can show. -The `web` command writes a graph of the profile data in SVG format and opens it in a web -browser. -(There is also a `gv` command that writes PostScript and opens it in Ghostview. -For either command, you need [[http://www.graphviz.org/][graphviz]] installed.) - - (pprof) web - -A small fragment of -[[http://benchgraffiti.googlecode.com/hg/havlak/havlak1.svg][the full graph]] looks like: - -.image profiling-go-programs_havlak1a-75.png - -Each box in the graph corresponds to a single function, and the boxes are sized -according to the number of samples in which the function was running. -An edge from box X to box Y indicates that X calls Y; the number along the edge is -the number of times that call appears in a sample. -If a call appears multiple times in a single sample, such as during recursive function -calls, each appearance counts toward the edge weight. -That explains the 21342 on the self-edge from `main.DFS` to itself. - -Just at a glance, we can see that the program spends much of its time in hash -operations, which correspond to use of Go's `map` values. -We can tell `web` to use only samples that include a specific function, such as -`runtime.mapaccess1_fast64`, which clears some of the noise from the graph: - - (pprof) web mapaccess1 - -.image profiling-go-programs_havlak1-hash_lookup-75.png - -If we squint, we can see that the calls to `runtime.mapaccess1_fast64` are being -made by `main.FindLoops` and `main.DFS`. - -Now that we have a rough idea of the big picture, it's time to zoom in on a particular -function. -Let's look at `main.DFS` first, just because it is a shorter function: - - (pprof) list DFS - Total: 2525 samples - ROUTINE ====================== main.DFS in /home/rsc/g/benchgraffiti/havlak/havlak1.go - 119 697 Total samples (flat / cumulative) - 3 3 240: func DFS(currentNode *BasicBlock, nodes []*UnionFindNode, number map[*BasicBlock]int, last []int, current int) int { - 1 1 241: nodes[current].Init(currentNode, current) - 1 37 242: number[currentNode] = current - . . 243: - 1 1 244: lastid := current - 89 89 245: for _, target := range currentNode.OutEdges { - 9 152 246: if number[target] == unvisited { - 7 354 247: lastid = DFS(target, nodes, number, last, lastid+1) - . . 248: } - . . 249: } - 7 59 250: last[number[currentNode]] = lastid - 1 1 251: return lastid - (pprof) - -The listing shows the source code for the `DFS` function (really, for every function -matching the regular expression `DFS`). -The first three columns are the number of samples taken while running that line, the -number of samples taken while running that line or in code called from that line, and -the line number in the file. -The related command `disasm` shows a disassembly of the function instead of a source -listing; when there are enough samples this can help you see which instructions are -expensive. -The `weblist` command mixes the two modes: it shows -[[http://benchgraffiti.googlecode.com/hg/havlak/havlak1.html][a source listing in which clicking a line shows the disassembly]]. - -Since we already know that the time is going into map lookups implemented by the -hash runtime functions, we care most about the second column. -A large fraction of time is spent in recursive calls to `DFS` (line 247), as would be -expected from a recursive traversal. -Excluding the recursion, it looks like the time is going into the accesses to the -`number` map on lines 242, 246, and 250. -For that particular lookup, a map is not the most efficient choice. -Just as they would be in a compiler, the basic block structures have unique sequence -numbers assigned to them. -Instead of using a `map[*BasicBlock]int` we can use a `[]int`, a slice indexed by the -block number. -There's no reason to use a map when an array or slice will do. - -Changing `number` from a map to a slice requires editing seven lines in the program -and cut its run time by nearly a factor of two: - - $ make havlak2 - go build havlak2.go - $ ./xtime ./havlak2 - # of loops: 76000 (including 1 artificial root node) - 16.55u 0.11s 16.69r 1321008kB ./havlak2 - $ - -(See the [[http://code.google.com/p/benchgraffiti/source/diff?name=34f7624bb2e2&r=240c155236f9&format=unidiff&path=/havlak/havlak.go][diff between `havlak1` and `havlak2`]]) - -We can run the profiler again to confirm that `main.DFS` is no longer a significant -part of the run time: - - $ make havlak2.prof - ./havlak2 -cpuprofile=havlak2.prof - # of loops: 76000 (including 1 artificial root node) - $ go tool pprof havlak2 havlak2.prof - Welcome to pprof! For help, type 'help'. - (pprof) - (pprof) top5 - Total: 1652 samples - 197 11.9% 11.9% 382 23.1% scanblock - 189 11.4% 23.4% 1549 93.8% main.FindLoops - 130 7.9% 31.2% 152 9.2% sweepspan - 104 6.3% 37.5% 896 54.2% runtime.mallocgc - 98 5.9% 43.5% 100 6.1% flushptrbuf - (pprof) - -The entry `main.DFS` no longer appears in the profile, and the rest of the program -runtime has dropped too. -Now the program is spending most of its time allocating memory and garbage collecting -(`runtime.mallocgc`, which both allocates and runs periodic garbage collections, -accounts for 54.2% of the time). -To find out why the garbage collector is running so much, we have to find out what is -allocating memory. -One way is to add memory profiling to the program. -We'll arrange that if the `-memprofile` flag is supplied, the program stops after one -iteration of the loop finding, writes a memory profile, and exits: - - var memprofile = flag.String("memprofile", "", "write memory profile to this file") - ... - - FindHavlakLoops(cfgraph, lsgraph) - if *memprofile != "" { - f, err := os.Create(*memprofile) - if err != nil { - log.Fatal(err) - } - pprof.WriteHeapProfile(f) - f.Close() - return - } - -We invoke the program with `-memprofile` flag to write a profile: - - $ make havlak3.mprof - go build havlak3.go - ./havlak3 -memprofile=havlak3.mprof - $ -(See the [[http://code.google.com/p/benchgraffiti/source/diff?name=240c155236f9&r=796913012f93&format=unidiff&path=/havlak/havlak.go][diff from havlak2]]) - -We use `go tool pprof` exactly the same way. Now the samples we are examining are -memory allocations, not clock ticks. - - $ go tool pprof havlak3 havlak3.mprof - Adjusting heap profiles for 1-in-524288 sampling rate - Welcome to pprof! For help, type 'help'. - (pprof) top5 - Total: 82.4 MB - 56.3 68.4% 68.4% 56.3 68.4% main.FindLoops - 17.6 21.3% 89.7% 17.6 21.3% main.(*CFG).CreateNode - 8.0 9.7% 99.4% 25.6 31.0% main.NewBasicBlockEdge - 0.5 0.6% 100.0% 0.5 0.6% itab - 0.0 0.0% 100.0% 0.5 0.6% fmt.init - (pprof) - -The command `go tool pprof` reports that `FindLoops` has allocated approximately -56.3 of the 82.4 MB in use; `CreateNode` accounts for another 17.6 MB. -To reduce overhead, the memory profiler only records information for approximately -one block per half megabyte allocated (the “1-in-524288 sampling rate”), so these -are approximations to the actual counts. - -To find the memory allocations, we can list those functions. - - (pprof) list FindLoops - Total: 82.4 MB - ROUTINE ====================== main.FindLoops in /home/rsc/g/benchgraffiti/havlak/havlak3.go - 56.3 56.3 Total MB (flat / cumulative) - ... - 1.9 1.9 268: nonBackPreds := make([]map[int]bool, size) - 5.8 5.8 269: backPreds := make([][]int, size) - . . 270: - 1.9 1.9 271: number := make([]int, size) - 1.9 1.9 272: header := make([]int, size, size) - 1.9 1.9 273: types := make([]int, size, size) - 1.9 1.9 274: last := make([]int, size, size) - 1.9 1.9 275: nodes := make([]*UnionFindNode, size, size) - . . 276: - . . 277: for i := 0; i < size; i++ { - 9.5 9.5 278: nodes[i] = new(UnionFindNode) - . . 279: } - ... - . . 286: for i, bb := range cfgraph.Blocks { - . . 287: number[bb.Name] = unvisited - 29.5 29.5 288: nonBackPreds[i] = make(map[int]bool) - . . 289: } - ... - -It looks like the current bottleneck is the same as the last one: using maps where -simpler data structures suffice. -`FindLoops` is allocating about 29.5 MB of maps. - -As an aside, if we run `go tool pprof` with the `--inuse_objects` flag, it will -report allocation counts instead of sizes: - - - $ go tool pprof --inuse_objects havlak3 havlak3.mprof - Adjusting heap profiles for 1-in-524288 sampling rate - Welcome to pprof! For help, type 'help'. - (pprof) list FindLoops - Total: 1763108 objects - ROUTINE ====================== main.FindLoops in /home/rsc/g/benchgraffiti/havlak/havlak3.go - 720903 720903 Total objects (flat / cumulative) - ... - . . 277: for i := 0; i < size; i++ { - 311296 311296 278: nodes[i] = new(UnionFindNode) - . . 279: } - . . 280: - . . 281: // Step a: - . . 282: // - initialize all nodes as unvisited. - . . 283: // - depth-first traversal and numbering. - . . 284: // - unreached BB's are marked as dead. - . . 285: // - . . 286: for i, bb := range cfgraph.Blocks { - . . 287: number[bb.Name] = unvisited - 409600 409600 288: nonBackPreds[i] = make(map[int]bool) - . . 289: } - ... - (pprof) - -Since the ~200,000 maps account for 29.5 MB, it looks like the initial map allocation -takes about 150 bytes. -That's reasonable when a map is being used to hold key-value pairs, but not when a map -is being used as a stand-in for a simple set, as it is here. - -Instead of using a map, we can use a simple slice to list the elements. -In all but one of the cases where maps are being used, it is impossible for the algorithm -to insert a duplicate element. -In the one remaining case, we can write a simple variant of the `append` built-in function: - - func appendUnique(a []int, x int) []int { - for _, y := range a { - if x == y { - return a - } - } - return append(a, x) - } - -In addition to writing that function, changing the Go program to use slices instead -of maps requires changing just a few lines of code. - - $ make havlak4 - go build havlak4.go - $ ./xtime ./havlak4 - # of loops: 76000 (including 1 artificial root node) - 11.84u 0.08s 11.94r 810416kB ./havlak4 - $ -(See the [[http://code.google.com/p/benchgraffiti/source/diff?name=796913012f93&r=d856c2f698c1&format=unidiff&path=/havlak/havlak.go][diff from havlak3]]) - -We're now at 2.11x faster than when we started. Let's look at a CPU profile again. - - $ make havlak4.prof - ./havlak4 -cpuprofile=havlak4.prof - # of loops: 76000 (including 1 artificial root node) - $ go tool pprof havlak4 havlak4.prof - Welcome to pprof! For help, type 'help'. - (pprof) top10 - Total: 1173 samples - 205 17.5% 17.5% 1083 92.3% main.FindLoops - 138 11.8% 29.2% 215 18.3% scanblock - 88 7.5% 36.7% 96 8.2% sweepspan - 76 6.5% 43.2% 597 50.9% runtime.mallocgc - 75 6.4% 49.6% 78 6.6% runtime.settype_flush - 74 6.3% 55.9% 75 6.4% flushptrbuf - 64 5.5% 61.4% 64 5.5% runtime.memmove - 63 5.4% 66.8% 524 44.7% runtime.growslice - 51 4.3% 71.1% 51 4.3% main.DFS - 50 4.3% 75.4% 146 12.4% runtime.MCache_Alloc - (pprof) - -Now memory allocation and the consequent garbage collection (`runtime.mallocgc`) -accounts for 50.9% of our run time. -Another way to look at why the system is garbage collecting is to look at the -allocations that are causing the collections, the ones that spend most of the time -in `mallocgc`: - - (pprof) web mallocgc - -.image profiling-go-programs_havlak4a-mallocgc.png - -It's hard to tell what's going on in that graph, because there are many nodes with -small sample numbers obscuring the big ones. -We can tell `go tool pprof` to ignore nodes that don't account for at least 10% of -the samples: - - $ go tool pprof --nodefraction=0.1 havlak4 havlak4.prof - Welcome to pprof! For help, type 'help'. - (pprof) web mallocgc - -.image profiling-go-programs_havlak4a-mallocgc-trim.png - -We can follow the thick arrows easily now, to see that `FindLoops` is triggering -most of the garbage collection. -If we list `FindLoops` we can see that much of it is right at the beginning: - - (pprof) list FindLoops - ... - . . 270: func FindLoops(cfgraph *CFG, lsgraph *LSG) { - . . 271: if cfgraph.Start == nil { - . . 272: return - . . 273: } - . . 274: - . . 275: size := cfgraph.NumNodes() - . . 276: - . 145 277: nonBackPreds := make([][]int, size) - . 9 278: backPreds := make([][]int, size) - . . 279: - . 1 280: number := make([]int, size) - . 17 281: header := make([]int, size, size) - . . 282: types := make([]int, size, size) - . . 283: last := make([]int, size, size) - . . 284: nodes := make([]*UnionFindNode, size, size) - . . 285: - . . 286: for i := 0; i < size; i++ { - 2 79 287: nodes[i] = new(UnionFindNode) - . . 288: } - ... - (pprof) - -Every time `FindLoops` is called, it allocates some sizable bookkeeping structures. -Since the benchmark calls `FindLoops` 50 times, these add up to a significant amount -of garbage, so a significant amount of work for the garbage collector. - -Having a garbage-collected language doesn't mean you can ignore memory allocation -issues. -In this case, a simple solution is to introduce a cache so that each call to `FindLoops` -reuses the previous call's storage when possible. -(In fact, in Hundt's paper, he explains that the Java program needed just this change to -get anything like reasonable performance, but he did not make the same change in the -other garbage-collected implementations.) - -We'll add a global `cache` structure: - - var cache struct { - size int - nonBackPreds [][]int - backPreds [][]int - number []int - header []int - types []int - last []int - nodes []*UnionFindNode - } - -and then have `FindLoops` consult it as a replacement for allocation: - - if cache.size < size { - cache.size = size - cache.nonBackPreds = make([][]int, size) - cache.backPreds = make([][]int, size) - cache.number = make([]int, size) - cache.header = make([]int, size) - cache.types = make([]int, size) - cache.last = make([]int, size) - cache.nodes = make([]*UnionFindNode, size) - for i := range cache.nodes { - cache.nodes[i] = new(UnionFindNode) - } - } - - nonBackPreds := cache.nonBackPreds[:size] - for i := range nonBackPreds { - nonBackPreds[i] = nonBackPreds[i][:0] - } - backPreds := cache.backPreds[:size] - for i := range nonBackPreds { - backPreds[i] = backPreds[i][:0] - } - number := cache.number[:size] - header := cache.header[:size] - types := cache.types[:size] - last := cache.last[:size] - nodes := cache.nodes[:size] - -Such a global variable is bad engineering practice, of course: it means that -concurrent calls to `FindLoops` are now unsafe. -For now, we are making the minimal possible changes in order to understand what -is important for the performance of our program; this change is simple and mirrors -the code in the Java implementation. -The final version of the Go program will use a separate `LoopFinder` instance to -track this memory, restoring the possibility of concurrent use. - - $ make havlak5 - go build havlak5.go - $ ./xtime ./havlak5 - # of loops: 76000 (including 1 artificial root node) - 8.03u 0.06s 8.11r 770352kB ./havlak5 - $ -(See the [[http://code.google.com/p/benchgraffiti/source/diff?name=d856c2f698c1&r=5ce46b0ee1db&format=unidiff&path=/havlak/havlak.go][diff from havlak4]]) - -There's more we can do to clean up the program and make it faster, but none of -it requires profiling techniques that we haven't already shown. -The work list used in the inner loop can be reused across iterations and across -calls to `FindLoops`, and it can be combined with the separate “node pool” generated -during that pass. -Similarly, the loop graph storage can be reused on each iteration instead of reallocated. -In addition to these performance changes, the -[[http://code.google.com/p/benchgraffiti/source/browse/havlak/havlak6.go][final version]] -is written using idiomatic Go style, using data structures and methods. -The stylistic changes have only a minor effect on the run time: the algorithm and -constraints are unchanged. - -The final version runs in 2.29 seconds and uses 351 MB of memory: - - $ make havlak6 - go build havlak6.go - $ ./xtime ./havlak6 - # of loops: 76000 (including 1 artificial root node) - 2.26u 0.02s 2.29r 360224kB ./havlak6 - $ - -That's 11 times faster than the program we started with. -Even if we disable reuse of the generated loop graph, so that the only cached memory -is the loop finding bookeeping, the program still runs 6.7x faster than the original -and uses 1.5x less memory. - - $ ./xtime ./havlak6 -reuseloopgraph=false - # of loops: 76000 (including 1 artificial root node) - 3.69u 0.06s 3.76r 797120kB ./havlak6 -reuseloopgraph=false - $ - -Of course, it's no longer fair to compare this Go program to the original C++ -program, which used inefficient data structures like `set`s where `vector`s would -be more appropriate. -As a sanity check, we translated the final Go program into -[[http://code.google.com/p/benchgraffiti/source/browse/havlak/havlak6.cc][equivalent C++ code]]. -Its execution time is similar to the Go program's: - - $ make havlak6cc - g++ -O3 -o havlak6cc havlak6.cc - $ ./xtime ./havlak6cc - # of loops: 76000 (including 1 artificial root node) - 1.99u 0.19s 2.19r 387936kB ./havlak6cc - -The Go program runs almost as fast as the C++ program. -As the C++ program is using automatic deletes and allocation instead of an explicit -cache, the C++ program a bit shorter and easier to write, but not dramatically so: - - $ wc havlak6.cc; wc havlak6.go - 401 1220 9040 havlak6.cc - 461 1441 9467 havlak6.go - $ -(See [[http://code.google.com/p/benchgraffiti/source/browse/havlak/havlak6.cc][havlak6.cc]] -and [[http://code.google.com/p/benchgraffiti/source/browse/havlak/havlak6.go][havlak6.go]]) - -Benchmarks are only as good as the programs they measure. -We used `go tool pprof` to study an inefficient Go program and then to improve its -performance by an order of magnitude and to reduce its memory usage by a factor of 3.7. -A subsequent comparison with an equivalently optimized C++ program shows that Go can be -competitive with C++ when programmers are careful about how much garbage is generated -by inner loops. - -The program sources, Linux x86-64 binaries, and profiles used to write this post -are available in the [[http://code.google.com/p/benchgraffiti/][benchgraffiti project on Google Code]]. - -As mentioned above, [[http://golang.org/cmd/go/#Test_packages][`go test`]] includes -these profiling flags already: define a -[[http://golang.org/pkg/testing/][benchmark function]] and you're all set. -There is also a standard HTTP interface to profiling data. In an HTTP server, adding - - import _ "net/http/pprof" - -will install handlers for a few URLs under `/debug/pprof/`. -Then you can run `go tool pprof` with a single argument—the URL to your server's -profiling data and it will download and examine a live profile. - - go tool pprof http://localhost:6060/debug/pprof/profile # 30-second CPU profile - go tool pprof http://localhost:6060/debug/pprof/heap # heap profile - go tool pprof http://localhost:6060/debug/pprof/block # goroutine blocking profile - -The goroutine blocking profile will be explained in a future post. Stay tuned. diff --git a/content/profiling-go-programs_havlak1-hash_lookup-75.png b/content/profiling-go-programs_havlak1-hash_lookup-75.png deleted file mode 100644 index 151fba25..00000000 Binary files a/content/profiling-go-programs_havlak1-hash_lookup-75.png and /dev/null differ diff --git a/content/profiling-go-programs_havlak1a-75.png b/content/profiling-go-programs_havlak1a-75.png deleted file mode 100644 index 35ca3a59..00000000 Binary files a/content/profiling-go-programs_havlak1a-75.png and /dev/null differ diff --git a/content/profiling-go-programs_havlak4a-mallocgc-trim.png b/content/profiling-go-programs_havlak4a-mallocgc-trim.png deleted file mode 100644 index b7b9c8d4..00000000 Binary files a/content/profiling-go-programs_havlak4a-mallocgc-trim.png and /dev/null differ diff --git a/content/profiling-go-programs_havlak4a-mallocgc.png b/content/profiling-go-programs_havlak4a-mallocgc.png deleted file mode 100644 index 1b390186..00000000 Binary files a/content/profiling-go-programs_havlak4a-mallocgc.png and /dev/null differ diff --git a/content/qihoo.article b/content/qihoo.article deleted file mode 100644 index 69aaaa83..00000000 --- a/content/qihoo.article +++ /dev/null @@ -1,101 +0,0 @@ -Qihoo 360 and Go -6 Jul 2015 - -Yang Zhou - -* Introduction - -_This_guest_blog_post_was_written_by_Yang_Zhou,_Software_Engineer_at_Qihoo_360._ - -[[http://www.360safe.com/][Qihoo 360]] is a major provider of Internet and -mobile security products and services in China, and operates a major -Android-based mobile distribution platform. At the end of June 2014, Qihoo had -about 500 million monthly active PC Internet users and over 640 million mobile -users. Qihoo also operates one of China’s most popular Internet browsers and PC -search engines. - -My team, the Push Service Team, provides fundamental messaging services for -more than 50 products across the company (both PC and mobile), including -thousands of Apps in our open platform. - -Our "love affair" with Go dates back to 2012 when we first attempted to provide -push services for one of Qihoo’s products. The initial version was built with -nginx + lua + redis, which failed to satisfy our requirement for real-time -performance due to excessive load. Under these circumstances, the -newly-published Go 1.0.3 release came to our attention. We completed a -prototype in a matter of weeks, largely thanks to the goroutine and channel -features it provided. - -Initially, our Go-based system ran on 20 servers, with 20 million real-time -connections in total. The system sent 2 million messages a day. That system now -runs on 400 servers, supporting 200 million+ real-time connections. It now -sends over 10 billion messages daily. - -With rapid business expansion and increasing application needs for our push -service, the initial Go system quickly reached its bottleneck: heap size went -up to 69G, with maximum garbage collection (GC) pauses of 3-6 seconds. Worse -still, we had to reboot the system every week to release memory. It wouldn’t be -honest if we didn’t consider relinquishing Go and instead, re-writing the -entire core component with C. However, things didn’t go exactly as we planned, -we ran into trouble migrating the code of Business Logic Layer. As a result, it -was impossible for the only personnel at that time (myself) to maintain the Go -system while ensuring the logic transfer to the C service framework. - -Therefore, I made the decision to stay with Go system (probably the wisest one -I had to make), and great headway was made soon enough. - -Here are a few tweaks we made and key take-aways: - -- Replace short connections with persistent ones (using a connection pool), to reduce creation of buffers and objects during communication. -- Use Objects and Memory pools appropriately, to reduce the load on the GC. - -.image qihoo/image00.png - -- Use a Task Pool, a mechanism with a group of long-lived goroutines consuming global task or message queues sent by connection goroutines, to replace short-lived goroutines. - -- Monitor and control goroutine numbers in the program. The lack of control can cause unbearable burden on the GC, imposed by surges in goroutines due to uninhibited acceptance of external requests, as RPC invocations sent to inner servers may block goroutines recently created. - -- Remember to add [[http://golang.org/pkg/net/#Conn][read and write deadlines]] to connections when under a mobile network; otherwise, it may lead to goroutine blockage. Apply it properly and with caution when under a LAN network, otherwise your RPC communication efficiency will be hurt. - -- Use Pipeline (under Full Duplex feature of TCP) to enhance the communication efficiency of RPC framework. - -As a result, we successfully launched three iterations of our architecture, and two iterations of our RPC framework even with limited human resources. This can all attributed to the development convenience of Go. Below you can find the up-to-date system architecture: - -.image qihoo/image01.png - -The continuous improvement journey can be illustrated by a table: - -.image qihoo/table.png - -Also, no temporary release of memory or system reboot is required after these -optimizations. - -What’s more exciting is we developed an on-line real-time Visibility Platform -for profiling Go programs. We can now easily access and diagnose the system -status, pinning down any potential risks. Here is a screen shot of the system -in action: - -.image qihoo/image02.png - -.image qihoo/image03.png - -The great thing about this platform is that we can actually simulate the -connection and behavior of millions of online users, by applying the -Distributed Stress Test Tool (also built using Go), and observe all real-time -visualized data. This allows us to evaluate the effectiveness of any -optimization and preclude problems by identifying system bottlenecks. - -Almost every possible system optimization has been practiced so far. And we -look forward to more good news from the GC team so that we could be further -relieved from heavy development work. I guess our experience may also grow -obsolete one day, as Go continues to evolve. - -This is why I want to conclude my sharing by extending my sincere appreciation -to the opportunity to attend [[http://gopherchina.org/][Gopher China]]. -It was a gala for us to learn, to share and for offering a window showcasing -Go’s popularity and prosperity in China. Many other teams within Qihoo have -already either got to know Go, or tried to use Go. - -I am convinced that many more Chinese Internet firms will join us in -re-creating their system in Go and the Go team's efforts will benefit more -developers and enterprises in the foreseeable future. diff --git a/content/qihoo/image00.png b/content/qihoo/image00.png deleted file mode 100644 index 0e570197..00000000 Binary files a/content/qihoo/image00.png and /dev/null differ diff --git a/content/qihoo/image01.png b/content/qihoo/image01.png deleted file mode 100644 index 7187ad24..00000000 Binary files a/content/qihoo/image01.png and /dev/null differ diff --git a/content/qihoo/image02.png b/content/qihoo/image02.png deleted file mode 100644 index e27f6a0a..00000000 Binary files a/content/qihoo/image02.png and /dev/null differ diff --git a/content/qihoo/image03.png b/content/qihoo/image03.png deleted file mode 100644 index 64be577e..00000000 Binary files a/content/qihoo/image03.png and /dev/null differ diff --git a/content/qihoo/table.png b/content/qihoo/table.png deleted file mode 100644 index 2da53111..00000000 Binary files a/content/qihoo/table.png and /dev/null differ diff --git a/content/race-detector.article b/content/race-detector.article deleted file mode 100644 index 49d5fee7..00000000 --- a/content/race-detector.article +++ /dev/null @@ -1,247 +0,0 @@ -Introducing the Go Race Detector -26 Jun 2013 -Tags: concurrency, technical - -Dmitry Vyukov - -Andrew Gerrand - -* Introduction - -[[http://en.wikipedia.org/wiki/Race_condition][Race conditions]] are among the -most insidious and elusive programming errors. They typically cause erratic and -mysterious failures, often long after the code has been deployed to production. -While Go's concurrency mechanisms make it easy to write clean concurrent code, -they don't prevent race conditions. Care, diligence, and testing are required. -And tools can help. - -We're happy to announce that Go 1.1 includes a -[[http://golang.org/doc/articles/race_detector.html][race detector]], -a new tool for finding race conditions in Go code. -It is currently available for Linux, OS X, and Windows systems -with 64-bit x86 processors. - -The race detector is based on the C/C++ -[[https://code.google.com/p/thread-sanitizer/][ThreadSanitizer runtime library]], -which has been used to detect many errors in Google's internal code base and in -[[http://www.chromium.org/][Chromium]]. -The technology was integrated with Go in September 2012; since then it has detected -[[https://code.google.com/p/go/issues/list?can=1&q=ThreadSanitizer][42 races]] -in the standard library. It is now part of our continuous build process, -where it continues to catch race conditions as they arise. - -* How it works - -The race detector is integrated with the go tool chain. When the -`-race` command-line flag is set, the compiler instruments all memory accesses -with code that records when and how the memory was accessed, while the runtime -library watches for unsynchronized accesses to shared variables. -When such "racy" behavior is detected, a warning is printed. -(See [[https://code.google.com/p/thread-sanitizer/wiki/Algorithm][this article]] -for the details of the algorithm.) - -Because of its design, the race detector can detect race conditions only when -they are actually triggered by running code, which means it's important to run -race-enabled binaries under realistic workloads. -However, race-enabled binaries can use ten times the CPU and memory, so it is -impractical to enable the race detector all the time. -One way out of this dilemma is to run some tests with the race detector -enabled. Load tests and integration tests are good candidates, since they tend -to exercise concurrent parts of the code. -Another approach using production workloads is to deploy a single race-enabled -instance within a pool of running servers. - -* Using the race detector - -The race detector is fully integrated with the Go tool chain. -To build your code with the race detector enabled, just add the -`-race` flag to the command line: - - $ go test -race mypkg    // test the package - $ go run -race mysrc.go  // compile and run the program - $ go build -race mycmd   // build the command - $ go install -race mypkg // install the package - -To try out the race detector for yourself, fetch and run this example program: - - $ go get -race code.google.com/p/go.blog/support/racy - $ racy - -* Examples - -Here are two examples of real issues caught by the race detector. - -** Example 1: Timer.Reset - -The first example is a simplified version of an actual bug found by the race -detector. It uses a timer to print a message after a random duration between 0 -and 1 second. It does so repeatedly for five seconds. -It uses [[http://golang.org/pkg/time/#AfterFunc][`time.AfterFunc`]] to create a -[[http://golang.org/pkg/time/#Timer][`Timer`]] for the first message and then -uses the [[http://golang.org/pkg/time/#Timer.Reset][`Reset`]] method to -schedule the next message, re-using the `Timer` each time. - -.play -numbers race-detector/timer.go /func main/,$ - -This looks like reasonable code, but under certain circumstances it fails in a surprising way: - - panic: runtime error: invalid memory address or nil pointer dereference - [signal 0xb code=0x1 addr=0x8 pc=0x41e38a] - - goroutine 4 [running]: - time.stopTimer(0x8, 0x12fe6b35d9472d96) - src/pkg/runtime/ztime_linux_amd64.c:35 +0x25 - time.(*Timer).Reset(0x0, 0x4e5904f, 0x1) - src/pkg/time/sleep.go:81 +0x42 - main.func·001() - race.go:14 +0xe3 - created by time.goFunc - src/pkg/time/sleep.go:122 +0x48 - -What's going on here? Running the program with the race detector enabled is more illuminating: - - ================== - WARNING: DATA RACE - Read by goroutine 5: - main.func·001() - race.go:14 +0x169 - - Previous write by goroutine 1: - main.main() - race.go:15 +0x174 - - Goroutine 5 (running) created at: - time.goFunc() - src/pkg/time/sleep.go:122 +0x56 - timerproc() - src/pkg/runtime/ztime_linux_amd64.c:181 +0x189 - ================== - -The race detector shows the problem: an unsynchronized read and write of the -variable `t` from different goroutines. If the initial timer duration is very -small, the timer function may fire before the main goroutine has assigned a -value to `t` and so the call to `t.Reset` is made with a nil `t`. - -To fix the race condition we change the code to read and write the variable -`t` only from the main goroutine: - -.play -numbers race-detector/timer-fixed.go /func main/,/^}/ - -Here the main goroutine is wholly responsible for setting and resetting the -`Timer` `t` and a new reset channel communicates the need to reset the timer in -a thread-safe way. - -A simpler but less efficient approach is to -[[http://play.golang.org/p/kuWTrY0pS4][avoid reusing timers]]. - -** Example 2: ioutil.Discard - -The second example is more subtle. - -The `ioutil` package's -[[http://golang.org/pkg/io/ioutil/#Discard][`Discard`]] object implements -[[http://golang.org/pkg/io/#Writer][`io.Writer`]], -but discards all the data written to it. -Think of it like `/dev/null`: a place to send data that you need to read but -don't want to store. -It is commonly used with [[http://golang.org/pkg/io/#Copy][`io.Copy`]] -to drain a reader, like this: - - io.Copy(ioutil.Discard, reader) - -Back in July 2011 the Go team noticed that using `Discard` in this way was -inefficient: the `Copy` function allocates an internal 32 kB buffer each time it -is called, but when used with `Discard` the buffer is unnecessary since we're -just throwing the read data away. -We thought that this idiomatic use of `Copy` and `Discard` should not be so costly. - -The fix was simple. -If the given `Writer` implements a `ReadFrom` method, a `Copy` call like this: - - io.Copy(writer, reader) - -is delegated to this potentially more efficient call: - - writer.ReadFrom(reader) - -We -[[https://code.google.com/p/go/source/detail?r=13faa632ba3a#][added a ReadFrom method]] -to Discard's underlying type, which has an internal buffer that is shared -between all its users. -We knew this was theoretically a race condition, but since all writes to the -buffer should be thrown away we didn't think it was important. - -When the race detector was implemented it immediately -[[https://code.google.com/p/go/issues/detail?id=3970][flagged this code]] as racy. -Again, we considered that the code might be problematic, but decided that the -race condition wasn't "real". -To avoid the "false positive" in our build we implemented -[[https://code.google.com/p/go/source/detail?r=1e55cf10aa4f][a non-racy version]] -that is enabled only when the race detector is running. - -But a few months later [[http://bradfitz.com/][Brad]] encountered a -[[https://code.google.com/p/go/issues/detail?id=4589][frustrating and strange bug]]. -After a few days of debugging, he narrowed it down to a real race condition -caused by `ioutil.Discard`. - -Here is the known-racy code in `io/ioutil`, where `Discard` is a -`devNull` that shares a single buffer between all of its users. - -.code race-detector/blackhole.go /var blackHole/,/^}/ - -Brad's program includes a `trackDigestReader` type, which wraps an `io.Reader` -and records the hash digest of what it reads. - - type trackDigestReader struct { - r io.Reader - h hash.Hash - } - - func (t trackDigestReader) Read(p []byte) (n int, err error) { - n, err = t.r.Read(p) - t.h.Write(p[:n]) - return - } - -For example, it could be used to compute the SHA-1 hash of a file while reading it: - - tdr := trackDigestReader{r: file, h: sha1.New()} - io.Copy(writer, tdr) - fmt.Printf("File hash: %x", tdr.h.Sum(nil)) - -In some cases there would be nowhere to write the data—but still a need to hash -the file—and so `Discard` would be used: - - io.Copy(ioutil.Discard, tdr) - -But in this case the `blackHole` buffer isn't just a black hole; it is a -legitimate place to store the data between reading it from the source -`io.Reader` and writing it to the `hash.Hash`. -With multiple goroutines hashing files simultaneously, each sharing the same -`blackHole` buffer, the race condition manifested itself by corrupting the data -between reading and hashing. -No errors or panics occurred, but the hashes were wrong. Nasty! - - func (t trackDigestReader) Read(p []byte) (n int, err error) { - // the buffer p is blackHole - n, err = t.r.Read(p) - // p may be corrupted by another goroutine here, - // between the Read above and the Write below - t.h.Write(p[:n]) - return - } - -The bug was finally -[[https://code.google.com/p/go/source/detail?r=4b61f121966b][fixed]] -by giving a unique buffer to each use of `ioutil.Discard`, eliminating the race -condition on the shared buffer. - -* Conclusions - -The race detector is a powerful tool for checking the correctness of concurrent -programs. It will not issue false positives, so take its warnings seriously. -But it is only as good as your tests; you must make sure they thoroughly -exercise the concurrent properties of your code so that the race detector can -do its job. - -What are you waiting for? Run `"go`test`-race"` on your code today! diff --git a/content/race-detector/blackhole.go b/content/race-detector/blackhole.go deleted file mode 100644 index d95912a1..00000000 --- a/content/race-detector/blackhole.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build OMIT - -package main - -var blackHole [4096]byte // shared buffer - -func (devNull) ReadFrom(r io.Reader) (n int64, err error) { - readSize := 0 - for { - readSize, err = r.Read(blackHole[:]) - n += int64(readSize) - if err != nil { - if err == io.EOF { - return n, nil - } - return - } - } -} diff --git a/content/race-detector/timer-fixed.go b/content/race-detector/timer-fixed.go deleted file mode 100644 index 279067bd..00000000 --- a/content/race-detector/timer-fixed.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "math/rand" - "time" -) - -func main() { - start := time.Now() - reset := make(chan bool) - var t *time.Timer - t = time.AfterFunc(randomDuration(), func() { - fmt.Println(time.Now().Sub(start)) - reset <- true - }) - for time.Since(start) < 5*time.Second { - <-reset - t.Reset(randomDuration()) - } -} - -func randomDuration() time.Duration { - return time.Duration(rand.Int63n(1e9)) -} diff --git a/content/race-detector/timer.go b/content/race-detector/timer.go deleted file mode 100644 index 298233db..00000000 --- a/content/race-detector/timer.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build OMIT - -package main - -import ( - "fmt" - "math/rand" - "time" -) - -func main() { - start := time.Now() - var t *time.Timer - t = time.AfterFunc(randomDuration(), func() { - fmt.Println(time.Now().Sub(start)) - t.Reset(randomDuration()) - }) - time.Sleep(5 * time.Second) -} - -func randomDuration() time.Duration { - return time.Duration(rand.Int63n(1e9)) -} diff --git a/content/real-go-projects-smarttwitter-and-webgo.article b/content/real-go-projects-smarttwitter-and-webgo.article deleted file mode 100644 index 1bb97265..00000000 --- a/content/real-go-projects-smarttwitter-and-webgo.article +++ /dev/null @@ -1,23 +0,0 @@ -Real Go Projects: SmartTwitter and web.go -19 Oct 2010 -Tags: guest - -Michael Hoisie - -* Introduction - -_This_week's_article_is_written_by_ [[http://www.hoisie.com/][_Michael_Hoisie_]]. _A_programmer_based_in_San_Francisco,_he_is_one_of_Go's_early_adopters_and_the_author_of_several_popular_Go_libraries._He_describes_his_experiences_using_Go:_ - -I was introduced to Go by a post on [[http://news.ycombinator.com/][Hacker News]]. About an hour later I was hooked. At the time I was working at a web start-up, and had been developing internal testing apps in Python. Go offered speed, better concurrency support, and sane Unicode handling, so I was keen to port my programs to the language. At that time there wasn't an easy way to write web apps in Go, so I decided to build a simple web framework, [[http://github.com/hoisie/web.go][web.go]]. It was modeled after a popular Python framework, [[http://webpy.org/][web.py]], which I had worked with previously. While working on web.go I got involved in the Go community, submitted a bunch of bug reports, and hacked on some standard library packages (mainly [[http://golang.org/pkg/http/][http]] and [[http://golang.org/pkg/json/][json]]). - -After a few weeks I noticed that web.go was getting attention at Github. This was surprising because I'd never really promoted the project. I think there's a niche for simple, fast web applications, and I think Go can fill it. - -One weekend I decided to write a simple Facebook application: it would re-post your Twitter status updates to your Facebook profile. There is an official Twitter application to do this, but it re-posts everything, creating noise in your Facebook feed. My application allowed you to filter retweets, mentions, hashtags, replies, and more. This turned into [[http://www.facebook.com/apps/application.php?id=135488932982][Smart Twitter]], which currently has nearly 90,000 users. - -The entire program is written in Go, and uses [[http://code.google.com/p/redis/][Redis]] as its storage back-end. It is very fast and robust. It currently processes about two dozen tweets per second, and makes heavy use of Go's channels. It runs on a single Virtual Private Server instance with 2GB of RAM, which has no problem handling the load. Smart Twitter uses very little CPU time, and is almost entirely memory-bound as the entire database is kept in memory. At any given time there are around 10 goroutines running concurrently: one accepting HTTP connections, another reading from the Twitter Streaming API, a couple for error handling, and the rest either processing web requests or re-posting incoming tweets. - -Smart Twitter also spawned other open-source Go projects: [[http://github.com/hoisie/mustache.go][mustache.go]], [[http://github.com/hoisie/redis.go][redis.go]], and [[http://github.com/hoisie/twitterstream][twitterstream]]. - -I see a lot of work left to do on web.go. For instance, I'd like to add better support for streaming connections, websockets, route filters, better support in shared hosts, and improving the documentation. I recently left the start-up to do software freelancing, and I'm planning to use Go where possible. This means I'll probably use it as a back end for personal apps, as well as for clients that like working with cutting edge technology. - -Finally, I'd like to thank the Go team for all their effort. Go is a wonderful platform and I think it has a bright future. I hope to see the language grow around the needs of the community. There's a lot of interesting stuff happening in the community, and I look forward to seeing what people can hack together with the language. diff --git a/content/share-memory-by-communicating.article b/content/share-memory-by-communicating.article deleted file mode 100644 index 144e6c34..00000000 --- a/content/share-memory-by-communicating.article +++ /dev/null @@ -1,79 +0,0 @@ -Share Memory By Communicating -13 Jul 2010 -Tags: concurrency, technical - -Andrew Gerrand - -* Introduction - -Traditional threading models (commonly used when writing Java, C++, and Python programs, for example) require the programmer to communicate between threads using shared memory. Typically, shared data structures are protected by locks, and threads will contend over those locks to access the data. In some cases, this is made easier by the use of thread-safe data structures such as Python's Queue. - -Go's concurrency primitives - goroutines and channels - provide an elegant and distinct means of structuring concurrent software. (These concepts have an [[http://swtch.com/~rsc/thread/][interesting history]] that begins with C. A. R. Hoare's [[http://www.usingcsp.com/][Communicating Sequential Processes]].) Instead of explicitly using locks to mediate access to shared data, Go encourages the use of channels to pass references to data between goroutines. This approach ensures that only one goroutine has access to the data at a given time. The concept is summarized in the document [[http://golang.org/doc/effective_go.html][Effective Go]] (a must-read for any Go programmer): - -_Do_not_communicate_by_sharing_memory;_instead,_share_memory_by_communicating._ - -Consider a program that polls a list of URLs. In a traditional threading environment, one might structure its data like so: - - type Resource struct { - url string - polling bool - lastPolled int64 - } - - type Resources struct { - data []*Resource - lock *sync.Mutex - } - -And then a Poller function (many of which would run in separate threads) might look something like this: - - func Poller(res *Resources) { - for { - // get the least recently-polled Resource - // and mark it as being polled - res.lock.Lock() - var r *Resource - for _, v := range res.data { - if v.polling { - continue - } - if r == nil || v.lastPolled < r.lastPolled { - r = v - } - } - if r != nil { - r.polling = true - } - res.lock.Unlock() - if r == nil { - continue - } - - // poll the URL - - // update the Resource's polling and lastPolled - res.lock.Lock() - r.polling = false - r.lastPolled = time.Nanoseconds() - res.lock.Unlock() - } - } - -This function is about a page long, and requires more detail to make it complete. It doesn't even include the URL polling logic (which, itself, would only be a few lines), nor will it gracefully handle exhausting the pool of Resources. - -Let's take a look at the same functionality implemented using Go idiom. In this example, Poller is a function that receives Resources to be polled from an input channel, and sends them to an output channel when they're done. - - type Resource string - - func Poller(in, out chan *Resource) { - for r := range in { - // poll the URL - - // send the processed Resource to out - out <- r - } - } - -The delicate logic from the previous example is conspicuously absent, and our Resource data structure no longer contains bookkeeping data. In fact, all that's left are the important parts. This should give you an inkling as to the power of these simple language features. - -There are many omissions from the above code snippets. For a walkthrough of a complete, idiomatic Go program that uses these ides, see the Codewalk [[http://golang.org/doc/codewalk/sharemem/][_Share_Memory_By_Communicating_]]. diff --git a/content/slices.article b/content/slices.article deleted file mode 100644 index 998d4ae3..00000000 --- a/content/slices.article +++ /dev/null @@ -1,576 +0,0 @@ -Arrays, slices (and strings): The mechanics of 'append' -26 Sep 2013 -Tags: array, slice, string, copy, append - -Rob Pike - -* Introduction - -One of the most common features of procedural programming languages is -the concept of an array. -Arrays seem like simple things but there are many questions that must be -answered when adding them to a language, such as: - -- fixed-size or variable-size? -- is the size part of the type? -- what do multidimensional arrays look like? -- does the empty array have meaning? - -The answers to these questions affect whether arrays are just -a feature of the language or a core part of its design. - -In the early development of Go, it took about a year to decide the answers -to these questions before the design felt right. -The key step was the introduction of _slices_, which built on fixed-size -_arrays_ to give a flexible, extensible data structure. -To this day, however, programmers new to Go often stumble over the way slices -work, perhaps because experience from other languages has colored their thinking. - -In this post we'll attempt to clear up the confusion. -We'll do so by building up the pieces to explain how the `append` built-in function -works, and why it works the way it does. - -* Arrays - -Arrays are an important building block in Go, but like the foundation of a building -they are often hidden below more visible components. -We must talk about them briefly before we move on to the more interesting, -powerful, and prominent idea of slices. - -Arrays are not often seen in Go programs because -the size of an array is part of its type, which limits its expressive power. - -The declaration - -.code slices/prog010.go /var buffer/ - -declares the variable `buffer`, which holds 256 bytes. -The type of `buffer` includes its size, `[256]byte`. -An array with 512 bytes would be of the distinct type `[512]byte`. - -The data associated with an array is just that: an array of elements. -Schematically, our buffer looks like this in memory, - - buffer: byte byte byte ... 256 times ... byte byte byte - -That is, the variable holds 256 bytes of data and nothing else. We can -access its elements with the familiar indexing syntax, `buffer[0]`, `buffer[1]`, -and so on through `buffer[255]`. (The index range 0 through 255 covers -256 elements.) Attempting to index `buffer` with a value outside this -range will crash the program. - -There is a built-in function called `len` that returns the number of elements -of an array or slice and also of a few other data types. -For arrays, it's obvious what `len` returns. -In our example, `len(buffer)` returns the fixed value 256. - -Arrays have their place—they are a good representation of a transformation -matrix for instance—but their most common purpose in Go is to hold storage -for a slice. - -* Slices: The slice header - -Slices are where the action is, but to use them well one must understand -exactly what they are and what they do. - -A slice is a data structure describing a contiguous section of an array -stored separately from the slice variable itself. -_A_slice_is_not_an_array_. -A slice _describes_ a piece of an array. - -Given our `buffer` array variable from the previous section, we could create -a slice that describes elements 100 through 150 (to be precise, 100 through 149, -inclusive) by _slicing_ the array: - -.code slices/prog010.go /var slice/ - -In that snippet we used the full variable declaration to be explicit. -The variable `slice` has type `[]byte`, pronounced "slice of bytes", -and is initialized from the array, called -`buffer`, by slicing elements 100 (inclusive) through 150 (exclusive). -The more idiomatic syntax would drop the type, which is set by the initializing expression: - - var slice = buffer[100:150] - -Inside a function we could use the short declaration form, - - slice := buffer[100:150] - -What exactly is this slice variable? -It's not quite the full story, but for now think of a -slice as a little data structure with two elements: a length and a pointer to an element -of a array. -You can think of it as being built like this behind the scenes: - - type sliceHeader struct { - Length int - ZerothElement *byte - } - - slice := sliceHeader{ - Length: 50, - ZerothElement: &buffer[100], - } - -Of course, this is just an illustration. -Despite what this snippet says that `sliceHeader` struct is not visible -to the programmer, and the type -of the element pointer depends on the type of the elements, -but this gives the general idea of the mechanics. - -So far we've used a slice operation on an array, but we can also slice a slice, like this: - - slice2 := slice[5:10] - -Just as before, this operation creates a new slice, in this case with elements -5 through 9 (inclusive) of the original slice, which means elements -105 through 109 of the original array. -The underlying `sliceHeader` struct for the `slice2` variable looks like -this: - - slice2 := sliceHeader{ - Length: 5, - ZerothElement: &buffer[105], - } - -Notice that this header still points to the same underlying array, stored in -the `buffer` variable. - -We can also _reslice_, which is to say slice a slice and store the result back in -the original slice structure. After - - slice = slice[5:10] - -the `sliceHeader` structure for the `slice` variable looks just like it did for the `slice2` -variable. -You'll see reslicing used often, for example to truncate a slice. This statement drops -the first and last elements of our slice: - - slice = slice[1:len(slice)-1] - -[Exercise: Write out what the `sliceHeader` struct looks like after this assignment.] - -You'll often hear experienced Go programmers talk about the "slice header" -because that really is what's stored in a slice variable. -For instance, when you call a function that takes a slice as an argument, such as -[[http://golang.org/pkg/bytes/#IndexRune][bytes.IndexRune]], that header is -what gets passed to the function. -In this call, - - slashPos := bytes.IndexRune(slice, '/') - -the `slice` argument that is passed to the `IndexRune` function is, in fact, -a "slice header". - -There's one more data item in the slice header, which we talk about below, -but first let's see what the existence of the slice header means when you -program with slices. - -* Passing slices to functions - -It's important to understand that even though a slice contains a pointer, -it is itself a value. -Under the covers, it is a struct value holding a pointer and a length. -It is _not_ a pointer to a struct. - -This matters. - -When we called `IndexRune` in the previous example, -it was passed a _copy_ of the slice header. -That behavior has important ramifications. - -Consider this simple function: - -.code slices/prog010.go /^func/,/^}/ - -It does just what its name implies, iterating over the indices of a slice -(using a `for` `range` loop), incrementing its elements. - -Try it: - -.play -edit slices/prog010.go /^func main/,/^}/ - -(You can edit and re-execute these runnable snippets if you want to explore.) - -Even though the slice _header_ is passed by value, the header includes -a pointer to elements of an array, so both the original slice header -and the copy of the header passed to the function describe the same -array. -Therefore, when the function returns, the modified elements can -be seen through the original slice variable. - -The argument to the function really is a copy, as this example shows: - -.play -edit slices/prog020.go /^func/,$ - -Here we see that the _contents_ of a slice argument can be modified by a function, -but its _header_ cannot. -The length stored in the `slice` variable is not modified by the call to the function, -since the function is passed a copy of the slice header, not the original. -Thus if we want to write a function that modifies the header, we must return it as a result -parameter, just as we have done here. -The `slice` variable is unchanged but the returned value has the new length, -which is then stored in `newSlice`, - -* Pointers to slices: Method receivers - -Another way to have a function modify the slice header is to pass a pointer to it. -Here's a variant of our previous example that does this: - -.play -edit slices/prog030.go /^func/,$ - -It seems clumsy in that example, especially dealing with the extra level of indirection -(a temporary variable helps), -but there is one common case where you see pointers to slices. -It is idiomatic to use a pointer receiver for a method that modifies a slice. - -Let's say we wanted to have a method on a slice that truncates it at the final slash. -We could write it like this: - -.play -edit slices/prog040.go /^type/,$ - -If you run this example you'll see that it works properly, updating the slice in the caller. - -[Exercise: Change the type of the receiver to be a value rather -than a pointer and run it again. Explain what happens.] - -On the other hand, if we wanted to write a method for `path` that upper-cases -the ASCII letters in the path (parochially ignoring non-English names), the method could -be a value because the value receiver will still point to the same underlying array. - -.play -edit slices/prog050.go /^type/,$ - -Here the `ToUpper` method uses two variables in the `for` `range` construct -to capture the index and slice element. -This form of loop avoids writing `p[i]` multiple times in the body. - -[Exercise: Convert the `ToUpper` method to use a pointer receiver and see if its behavior changes.] - -[Advanced exercise: Convert the `ToUpper` method to handle Unicode letters, not just ASCII.] - -* Capacity - -Look at the following function that extends its argument slice of `ints` by one element: - -.code slices/prog060.go /^func Extend/,/^}/ - -(Why does it need to return the modified slice?) Now run it: - -.play -edit slices/prog060.go /^func main/,/^}/ - -See how the slice grows until... it doesn't. - -It's time to talk about the third component of the slice header: its _capacity_. -Besides the array pointer and length, the slice header also stores its capacity: - - type sliceHeader struct { - Length int - Capacity int - ZerothElement *byte - } - -The `Capacity` field records how much space the underlying array actually has; it is the maximum -value the `Length` can reach. -Trying to grow the slice beyond its capacity will step beyond the limits of the array and will trigger a panic. - -After our example slice is created by - - slice := iBuffer[0:0] - -its header looks like this: - - slice := sliceHeader{ - Length: 0, - Capacity: 10, - ZerothElement: &iBuffer[0], - } - -The `Capacity` field is equal to the length of the underlying array, -minus the index in the array of the first element of the slice (zero in this case). -If you want to inquire what the capacity is for a slice, use the built-in function `cap`: - - if cap(slice) == len(slice) { - fmt.Println("slice is full!") - } - -* Make - -What if we want to grow the slice beyond its capacity? -You can't! -By definition, the capacity is the limit to growth. -But you can achieve an equivalent result by allocating a new array, copying the data over, and modifying -the slice to describe the new array. - -Let's start with allocation. -We could use the `new` built-in function to allocate a bigger array -and then slice the result, -but it is simpler to use the `make` built-in function instead. -It allocates a new array and -creates a slice header to describe it, all at once. -The `make` function takes three arguments: the type of the slice, its initial length, and its capacity, which is the -length of the array that `make` allocates to hold the slice data. -This call creates a slice of length 10 with room for 5 more (15-10), as you can see by running it: - -.play -edit slices/prog070.go /slice/,/fmt/ - -This snippet doubles the capacity of our `int` slice but keeps its length the same: - -.play -edit slices/prog080.go /slice/,/OMIT/ - -After running this code the slice has much more room to grow before needing another reallocation. - -When creating slices, it's often true that the length and capacity will be same. -The `make` built-in has a shorthand for this common case. -The length argument defaults to the capacity, so you can leave it out -to set them both to the same value. -After - - gophers := make([]Gopher, 10) - -the `gophers` slice has both its length and capacity set to 10. - -* Copy - -When we doubled the capacity of our slice in the previous section, -we wrote a loop to copy the old data to the new slice. -Go has a built-in function, `copy`, to make this easier. -Its arguments are two slices, and it copies the data from the right-hand argument to the left-hand argument. -Here's our example rewritten to use `copy`: - -.play -edit slices/prog090.go /newSlice/,/newSlice/ - -The `copy` function is smart. -It only copies what it can, paying attention to the lengths of both arguments. -In other words, the number of elements it copies is the minimum of the lengths of the two slices. -This can save a little bookkeeping. -Also, `copy` returns an integer value, the number of elements it copied, although it's not always worth checking. - -The `copy` function also gets things right when source and destination overlap, which means it can be used to shift -items around in a single slice. -Here's how to use `copy` to insert a value into the middle of a slice. - -.code slices/prog100.go /Insert/,/^}/ - -There are a couple of things to notice in this function. -First, of course, it must return the updated slice because its length has changed. -Second, it uses a convenient shorthand. -The expression - - slice[i:] - -means exactly the same as - - slice[i:len(slice)] - -Also, although we haven't used the trick yet, we can leave out the first element of a slice expression too; -it defaults to zero. Thus - - slice[:] - -just means the slice itself, which is useful when slicing an array. -This expression is the shortest way to say "a slice describing all the elements of the array": - - array[:] - -Now that's out of the way, let's run our `Insert` function. - -.play -edit slices/prog100.go /make/,/OMIT/ - -* Append: An example - -A few sections back, we wrote an `Extend` function that extends a slice by one element. -It was buggy, though, because if the slice's capacity was too small, the function would -crash. -(Our `Insert` example has the same problem.) -Now we have the pieces in place to fix that, so let's write a robust implementation of -`Extend` for integer slices. - -.code slices/prog110.go /func Extend/,/^}/ - -In this case it's especially important to return the slice, since when it reallocates -the resulting slice describes a completely different array. -Here's a little snippet to demonstrate what happens as the slice fills up: - -.play -edit slices/prog110.go /START/,/END/ - -Notice the reallocation when the initial array of size 5 is filled up. -Both the capacity and the address of the zeroth element change when the new array is allocated. - -With the robust `Extend` function as a guide we can write an even nicer function that lets -us extend the slice by multiple elements. -To do this, we use Go's ability to turn a list of function arguments into a slice when the -function is called. -That is, we use Go's variadic function facility. - -Let's call the function `Append`. -For the first version, we can just call `Extend` repeatedly so the mechanism of the variadic function is clear. -The signature of `Append` is this: - - func Append(slice []int, items ...int) []int - -What that says is that `Append` takes one argument, a slice, followed by zero or more -`int` arguments. -Those arguments are exactly a slice of `int` as far as the implementation -of `Append` is concerned, as you can see: - -.code slices/prog120.go /Append/,/^}/ - -Notice the `for` `range` loop iterating over the elements of the `items` argument, which has implied type `[]int`. -Also notice the use of the blank identifier `_` to discard the index in the loop, which we don't need in this case. - -Try it: - -.play -edit slices/prog120.go /START/,/END/ - -Another new technique is in this example is that we initialize the slice by writing a composite literal, -which consists of the type of the slice followed by its elements in braces: - -.code slices/prog120.go /slice := / - -The `Append` function is interesting for another reason. -Not only can we append elements, we can append a whole second slice -by "exploding" the slice into arguments using the `...` notation at the call site: - -.play -edit slices/prog130.go /START/,/END/ - -Of course, we can make `Append` more efficient by allocating no more than once, -building on the innards of `Extend`: - -.code slices/prog140.go /Append/,/^}/ - -Here, notice how we use `copy` twice, once to move the slice data to the newly -allocated memory, and then to copy the appending items to the end of the old data. - -Try it; the behavior is the same as before: - -.play -edit slices/prog140.go /START/,/END/ - -* Append: The built-in function - -And so we arrive at the motivation for the design of the `append` built-in function. -It does exactly what our `Append` example does, with equivalent efficiency, but it -works for any slice type. - -A weakness of Go is that any generic-type operations must be provided by the -run-time. Some day that may change, but for now, to make working with slices -easier, Go provides a built-in generic `append` function. -It works the same as our `int` slice version, but for _any_ slice type. - -Remember, since the slice header is always updated by a call to `append`, you need -to save the returned slice after the call. -In fact, the compiler won't let you call append without saving the result. - -Here are some one-liners intermingled with print statements. Try them, edit them and explore: - -.play -edit slices/prog150.go /START/,/END/ - -It's worth taking a moment to think about the final one-liner of that example in detail to understand -how the design of slices makes it possible for this simple call to work correctly. - -There are lots more examples of `append`, `copy`, and other ways to use slices -on the community-built -[[https://golang.org/wiki/SliceTricks]["Slice Tricks" Wiki page]]. - -* Nil - -As an aside, with our newfound knowledge we can see what the representation of a `nil` slice is. -Naturally, it is the zero value of the slice header: - - sliceHeader{ - Length: 0, - Capacity: 0, - ZerothElement: nil, - } - -or just - - sliceHeader{} - -The key detail is that the element pointer is `nil` too. The slice created by - - array[0:0] - -has length zero (and maybe even capacity zero) but its pointer is not `nil`, so -it is not a nil slice. - -As should be clear, an empty slice can grow (assuming it has non-zero capacity), but a `nil` -slice has no array to put values in and can never grow to hold even one element. - -That said, a `nil` slice is functionally equivalent to a zero-length slice, even though it points -to nothing. -It has length zero and can be appended to, with allocation. -As an example, look at the one-liner above that copies a slice by appending -to a `nil` slice. - -* Strings - -Now a brief section about strings in Go in the context of slices. - -Strings are actually very simple: they are just read-only slices of bytes with a bit -of extra syntactic support from the language. - -Because they are read-only, there is no need for a capacity (you can't grow them), -but otherwise for most purposes you can treat them just like read-only slices -of bytes. - -For starters, we can index them to access individual bytes: - - slash := "/usr/ken"[0] // yields the byte value '/'. - -We can slice a string to grab a substring: - - usr := "/usr/ken"[0:4] // yields the string "/usr" - -It should be obvious now what's going on behind the scenes when we slice a string. - -We can also take a normal slice of bytes and create a string from it with the simple conversion: - - str := string(slice) - -and go in the reverse direction as well: - - slice := []byte(usr) - -The array underlying a string is hidden from view; there is no way to access its contents -except through the string. That means that when we do either of these conversions, a -copy of the array must be made. -Go takes care of this, of course, so you don't have to. -After either of these conversions, modifications to -the array underlying the byte slice don't affect the corresponding string. - -An important consequence of this slice-like design for strings is that -creating a substring is very efficient. -All that needs to happen -is the creation of a two-word string header. Since the string is read-only, the original -string and the string resulting from the slice operation can share the same array safely. - -A historical note: The earliest implementation of strings always allocated, but when slices -were added to the language, they provided a model for efficient string handling. Some of -the benchmarks saw huge speedups as a result. - -There's much more to strings, of course, but they are a topic for another post. - -* Conclusion - -To understand how slices work, it helps to understand how they are implemented. -There is a little data structure, the slice header, that is the item associated with the slice -variable, and that header describes a section of a separately allocated array. -When we pass slice values around, the header gets copied but the array it points -to is always shared. - -Once you appreciate how they work, slices become not only easy to use, but -powerful and expressive, especially with the help of the `copy` and `append` -built-in functions. - -* More reading - -There's lots to find around the intertubes about slices in Go. -As mentioned earlier, -the [[https://golang.org/wiki/SliceTricks]["Slice Tricks" Wiki page]] -has many examples. -The [[http://blog.golang.org/go-slices-usage-and-internals][Go Slices]] blog post -describes the memory layout details with clear diagrams. -Russ Cox's [[http://research.swtch.com/godata][Go Data Structures]] article includes -a discussion of slices along with some of Go's other internal data structures. - -There is much more material available, but the best way to learn about slices is to use them. diff --git a/content/slices/prog010.go b/content/slices/prog010.go deleted file mode 100644 index 5fffa11d..00000000 --- a/content/slices/prog010.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -var buffer [256]byte -var slice []byte = buffer[100:150] - -func AddOneToEachElement(slice []byte) { - for i := range slice { - slice[i]++ - } -} - -func main() { - slice := buffer[10:20] - for i := 0; i < len(slice); i++ { - slice[i] = byte(i) - } - fmt.Println("before", slice) - AddOneToEachElement(slice) - fmt.Println("after", slice) -} diff --git a/content/slices/prog020.go b/content/slices/prog020.go deleted file mode 100644 index 548225ce..00000000 --- a/content/slices/prog020.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -var buffer [256]byte -var slice []byte = buffer[100:150] - -func SubtractOneFromLength(slice []byte) []byte { - slice = slice[0 : len(slice)-1] - return slice -} - -func main() { - fmt.Println("Before: len(slice) =", len(slice)) - newSlice := SubtractOneFromLength(slice) - fmt.Println("After: len(slice) =", len(slice)) - fmt.Println("After: len(newSlice) =", len(newSlice)) -} diff --git a/content/slices/prog030.go b/content/slices/prog030.go deleted file mode 100644 index 426ef04b..00000000 --- a/content/slices/prog030.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -var buffer [256]byte -var slice []byte = buffer[100:150] - -func PtrSubtractOneFromLength(slicePtr *[]byte) { - slice := *slicePtr - *slicePtr = slice[0 : len(slice)-1] -} - -func main() { - fmt.Println("Before: len(slice) =", len(slice)) - PtrSubtractOneFromLength(&slice) - fmt.Println("After: len(slice) =", len(slice)) -} diff --git a/content/slices/prog040.go b/content/slices/prog040.go deleted file mode 100644 index a05e7657..00000000 --- a/content/slices/prog040.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "fmt" -) - -type path []byte - -func (p *path) TruncateAtFinalSlash() { - i := bytes.LastIndex(*p, []byte("/")) - if i >= 0 { - *p = (*p)[0:i] - } -} - -func main() { - pathName := path("/usr/bin/tso") // Conversion from string to path. - pathName.TruncateAtFinalSlash() - fmt.Printf("%s\n", pathName) -} diff --git a/content/slices/prog050.go b/content/slices/prog050.go deleted file mode 100644 index 3994e6b2..00000000 --- a/content/slices/prog050.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -type path []byte - -func (p path) ToUpper() { - for i, b := range p { - if 'a' <= b && b <= 'z' { - p[i] = b + 'A' - 'a' - } - } -} - -func main() { - pathName := path("/usr/bin/tso") - pathName.ToUpper() - fmt.Printf("%s\n", pathName) -} diff --git a/content/slices/prog060.go b/content/slices/prog060.go deleted file mode 100644 index 0fdf9e60..00000000 --- a/content/slices/prog060.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -func Extend(slice []int, element int) []int { - n := len(slice) - slice = slice[0 : n+1] - slice[n] = element - return slice -} - -func main() { - var iBuffer [10]int - slice := iBuffer[0:0] - for i := 0; i < 20; i++ { - slice = Extend(slice, i) - fmt.Println(slice) - } -} diff --git a/content/slices/prog070.go b/content/slices/prog070.go deleted file mode 100644 index 782848a7..00000000 --- a/content/slices/prog070.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -func main() { - slice := make([]int, 10, 15) - fmt.Printf("len: %d, cap: %d\n", len(slice), cap(slice)) -} diff --git a/content/slices/prog080.go b/content/slices/prog080.go deleted file mode 100644 index 3886288e..00000000 --- a/content/slices/prog080.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -func main() { - slice := make([]int, 10, 15) - fmt.Printf("len: %d, cap: %d\n", len(slice), cap(slice)) - newSlice := make([]int, len(slice), 2*cap(slice)) - for i := range slice { - newSlice[i] = slice[i] - } - slice = newSlice - fmt.Printf("len: %d, cap: %d\n", len(slice), cap(slice)) - // OMIT -} diff --git a/content/slices/prog090.go b/content/slices/prog090.go deleted file mode 100644 index 5cb410c4..00000000 --- a/content/slices/prog090.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -func main() { - slice := make([]int, 10, 15) - fmt.Printf("len: %d, cap: %d\n", len(slice), cap(slice)) - newSlice := make([]int, len(slice), 2*cap(slice)) - copy(newSlice, slice) - slice = newSlice - fmt.Printf("len: %d, cap: %d\n", len(slice), cap(slice)) -} diff --git a/content/slices/prog100.go b/content/slices/prog100.go deleted file mode 100644 index 4e61725b..00000000 --- a/content/slices/prog100.go +++ /dev/null @@ -1,36 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -// Insert inserts the value into the slice at the specified index, -// which must be in range. -// The slice must have room for the new element. -func Insert(slice []int, index, value int) []int { - // Grow the slice by one element. - slice = slice[0 : len(slice)+1] - // Use copy to move the upper part of the slice out of the way and open a hole. - copy(slice[index+1:], slice[index:]) - // Store the new value. - slice[index] = value - // Return the result. - return slice -} - -func main() { - slice := make([]int, 10, 20) // Note capacity > length: room to add element. - for i := range slice { - slice[i] = i - } - fmt.Println(slice) - slice = Insert(slice, 5, 99) - fmt.Println(slice) - // OMIT -} diff --git a/content/slices/prog110.go b/content/slices/prog110.go deleted file mode 100644 index 25036023..00000000 --- a/content/slices/prog110.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -// Extend extends the slice by adding the element to the end. -func Extend(slice []int, element int) []int { - n := len(slice) - if n == cap(slice) { - // Slice is full; must grow. - // We double its size and add 1, so if the size is zero we still grow. - newSlice := make([]int, len(slice), 2*len(slice)+1) - copy(newSlice, slice) - slice = newSlice - } - slice = slice[0 : n+1] - slice[n] = element - return slice -} - -func main() { - // START OMIT - slice := make([]int, 0, 5) - for i := 0; i < 10; i++ { - slice = Extend(slice, i) - fmt.Printf("len=%d cap=%d slice=%v\n", len(slice), cap(slice), slice) - fmt.Println("address of 0th element:", &slice[0]) - } - // END OMIT -} diff --git a/content/slices/prog120.go b/content/slices/prog120.go deleted file mode 100644 index d9a4b2ea..00000000 --- a/content/slices/prog120.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -// Extend extends the slice by adding the element to the end. -func Extend(slice []int, element int) []int { - n := len(slice) - if n == cap(slice) { - // Slice is full; must grow. - // We double its size and add 1, so if the size is zero we still grow. - newSlice := make([]int, len(slice), 2*len(slice)+1) - copy(newSlice, slice) - slice = newSlice - } - slice = slice[0 : n+1] - slice[n] = element - return slice -} - -// Append appends the items to the slice. -// First version: just loop calling Extend. -func Append(slice []int, items ...int) []int { - for _, item := range items { - slice = Extend(slice, item) - } - return slice -} - -func main() { - // START1 OMIT - slice := []int{0, 1, 2, 3, 4} - fmt.Println(slice) - slice = Append(slice, 5, 6, 7, 8) - fmt.Println(slice) - // END1 OMIT -} diff --git a/content/slices/prog130.go b/content/slices/prog130.go deleted file mode 100644 index de3e009f..00000000 --- a/content/slices/prog130.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -// Extend extends the slice by adding the element to the end. -func Extend(slice []int, element int) []int { - n := len(slice) - if n == cap(slice) { - // Slice is full; must grow. - // We double its size and add 1, so if the size is zero we still grow. - newSlice := make([]int, len(slice), 2*len(slice)+1) - copy(newSlice, slice) - slice = newSlice - } - slice = slice[0 : n+1] - slice[n] = element - return slice -} - -// Append appends the items to the slice. -// First version: just loop calling Extend. -func Append(slice []int, items ...int) []int { - for _, item := range items { - slice = Extend(slice, item) - } - return slice -} - -func main() { - // START OMIT - slice1 := []int{0, 1, 2, 3, 4} - slice2 := []int{55, 66, 77} - fmt.Println(slice1) - slice1 = Append(slice1, slice2...) // The '...' is essential! - fmt.Println(slice1) - // END OMIT -} diff --git a/content/slices/prog140.go b/content/slices/prog140.go deleted file mode 100644 index 6a2b9c79..00000000 --- a/content/slices/prog140.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -// Append appends the elements to the slice. -// Efficient version. -func Append(slice []int, elements ...int) []int { - n := len(slice) - total := len(slice) + len(elements) - if total > cap(slice) { - // Reallocate. Grow to 1.5 times the new size, so we can still grow. - newSize := total*3/2 + 1 - newSlice := make([]int, total, newSize) - copy(newSlice, slice) - slice = newSlice - } - slice = slice[:total] - copy(slice[n:], elements) - return slice -} - -func main() { - // START OMIT - slice1 := []int{0, 1, 2, 3, 4} - slice2 := []int{55, 66, 77} - fmt.Println(slice1) - slice1 = Append(slice1, slice2...) // The '...' is essential! - fmt.Println(slice1) - // END OMIT -} diff --git a/content/slices/prog150.go b/content/slices/prog150.go deleted file mode 100644 index f73bfa7c..00000000 --- a/content/slices/prog150.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" -) - -func main() { - // START OMIT - // Create a couple of starter slices. - slice := []int{1, 2, 3} - slice2 := []int{55, 66, 77} - fmt.Println("Start slice: ", slice) - fmt.Println("Start slice2:", slice2) - - // Add an item to a slice. - slice = append(slice, 4) - fmt.Println("Add one item:", slice) - - // Add one slice to another. - slice = append(slice, slice2...) - fmt.Println("Add one slice:", slice) - - // Make a copy of a slice (of int). - slice3 := append([]int(nil), slice...) - fmt.Println("Copy a slice:", slice3) - - // Copy a slice to the end of itself. - fmt.Println("Before append to self:", slice) - slice = append(slice, slice...) - fmt.Println("After append to self:", slice) - // END OMIT -} diff --git a/content/spotlight-on-external-go-libraries.article b/content/spotlight-on-external-go-libraries.article deleted file mode 100644 index 6daf4cd5..00000000 --- a/content/spotlight-on-external-go-libraries.article +++ /dev/null @@ -1,41 +0,0 @@ -Spotlight on external Go libraries -3 Jun 2011 -Tags: community, libraries - -Andrew Gerrand - -* Introduction - -While the Go authors have been working hard at improving Go's standard library, the greater community has created a growing ecosystem of external libraries. In this post we look at some popular Go libraries and how they can be used. - -[[http://labix.org/mgo][Mgo]] (pronounced "mango") is a MongoDB database driver. [[http://www.mongodb.org/][MongoDB]] is a [[http://en.wikipedia.org/wiki/Document-oriented_database][document-oriented database]] with a long list of features suitable for [[http://www.mongodb.org/display/DOCS/Use%2BCases][a broad range of uses]]. The mgo package provides a rich, idiomatic Go API for working with MongoDB, from basic operations such as inserting and updating records to the more advanced [[http://www.mongodb.org/display/DOCS/MapReduce][MapReduce]] and [[http://www.mongodb.org/display/DOCS/GridFS][GridFS]] features. Mgo has a bunch of cool features including automated cluster discovery and result pre-fetching - see the [[http://labix.org/mgo][mgo homepage]] for details and example code. For working with large data sets Go, MongoDB, and mgo are a powerful combination. - -[[https://github.com/dchest/authcookie][Authcookie]] is a web library for generating and verifying user authentication cookies. It allows web servers to hand out cryptographically secure tokens tied to a specific user that will expire after a specified time period. It has a simple API that makes it straightforward to add authentication to existing web applications. See the [[https://github.com/dchest/authcookie/blob/master/README.md][README file]] for details and example code. - -[[http://code.google.com/p/go-charset][Go-charset]] provides support for converting between Go's standard UTF-8 encoding and a variety of character sets. The go-charset package implements a translating io.Reader and io.Writer so you can wrap existing Readers and Writers (such as network connections or file descriptors), making it easy to communicate with systems that use other character encodings. - -[[https://github.com/madari/go-socket.io][Go-socket.io]] is a Go implementation of [[http://socket.io/][Socket.IO]], a client/server API that allows web servers to push messages to web browsers. Depending on the capabilities of the user's browser, Socket.IO uses the best transport for the connection, be it modern websockets, AJAX long polling, or some [[http://socket.io/#transports][other mechanism]]. Go-socket.io bridges the gap between Go servers and rich JavaScript clients for a wide range of browsers. To get a feel for go-socket.io see the [[https://github.com/madari/go-socket.io/blob/master/example/example.go][chat server example]]. - -It's worth mentioning that these packages are [[http://golang.org/cmd/goinstall/][goinstallable]]. With an up-to-date Go [[http://golang.org/doc/install.html][installation]] you can install them all with a single command: - - goinstall launchpad.net/mgo \ - github.com/dchest/authcookie \ - go-charset.googlecode.com/hg/charset \ - github.com/madari/go-socket.io - - -Once goinstalled, the packages can be imported using those same paths: - - import ( - "launchpad.net/mgo" - "github.com/dchest/authcookie" - "go-charset.googlecode.com/hg/charset" - "github.com/madari/go-socket.io" - ) - - -Also, as they are now a part of the local Go system, we can inspect their documentation with [[http://golang.org/cmd/godoc/][godoc]]: - - godoc launchpad.net/mgo Database # see docs for Database type - -Of course, this is just the tip of the iceberg; there are more great Go libraries listed on the [[http://godashboard.appspot.com/package][package dashboard]] and many more to come. diff --git a/content/strings.article b/content/strings.article deleted file mode 100644 index 377945d1..00000000 --- a/content/strings.article +++ /dev/null @@ -1,331 +0,0 @@ -Strings, bytes, runes and characters in Go -23 Oct 2013 -Tags: strings, bytes, runes, characters - -Rob Pike - -* Introduction - -The [[http://blog.golang.org/slices][previous blog post]] explained how slices -work in Go, using a number of examples to illustrate the mechanism behind -their implementation. -Building on that background, this post discusses strings in Go. -At first, strings might seem too simple a topic for a blog post, but to use -them well requires understanding not only how they work, -but also the difference between a byte, a character, and a rune, -the difference between Unicode and UTF-8, -the difference between a string and a string literal, -and other even more subtle distinctions. - -One way to approach this topic is to think of it as an answer to the frequently -asked question, "When I index a Go string at position _n_, why don't I get the -_nth_ character?" -As you'll see, this question leads us to many details about how text works -in the modern world. - -An excellent introduction to some of these issues, independent of Go, -is Joel Spolsky's famous blog post, -[[http://www.joelonsoftware.com/articles/Unicode.html][The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode and Character Sets (No Excuses!)]]. -Many of the points he raises will be echoed here. - -* What is a string? - -Let's start with some basics. - -In Go, a string is in effect a read-only slice of bytes. -If you're at all uncertain about what a slice of bytes is or how it works, -please read the [[http://blog.golang.org/slices][previous blog post]]; -we'll assume here that you have. - -It's important to state right up front that a string holds _arbitrary_ bytes. -It is not required to hold Unicode text, UTF-8 text, or any other predefined format. -As far as the content of a string is concerned, it is exactly equivalent to a -slice of bytes. - -Here is a string literal (more about those soon) that uses the -`\xNN` notation to define a string constant holding some peculiar byte values. -(Of course, bytes range from hexadecimal values 00 through FF, inclusive.) - -.code strings/basic.go /const sample/ - -* Printing strings - -Because some of the bytes in our sample string are not valid ASCII, not even -valid UTF-8, printing the string directly will produce ugly output. -The simple print statement - -.code strings/basic.go /println/,/println/ - -produces this mess (whose exact appearance varies with the environment): - - ��=� ⌘ - -To find out what that string really holds, we need to take it apart and examine the pieces. -There are several ways to do this. -The most obvious is to loop over its contents and pull out the bytes -individually, as in this `for` loop: - -.code strings/basic.go /byte loop/,/byte loop/ - -As implied up front, indexing a string accesses individual bytes, not -characters. We'll return to that topic in detail below. For now, let's -stick with just the bytes. -This is the output from the byte-by-byte loop: - - bd b2 3d bc 20 e2 8c 98 - -Notice how the individual bytes match the -hexadecimal escapes that defined the string. - -A shorter way to generate presentable output for a messy string -is to use the `%x` (hexadecimal) format verb of `fmt.Printf`. -It just dumps out the sequential bytes of the string as hexadecimal -digits, two per byte. - -.code strings/basic.go /percent x/,/percent x/ - -Compare its output to that above: - - bdb23dbc20e28c98 - -A nice trick is to use the "space" flag in that format, putting a -space between the `%` and the `x`. Compare the format string -used here to the one above, - -.code strings/basic.go /percent space x/,/percent space x/ - -and notice how the bytes come -out with spaces between, making the result a little less imposing: - - bd b2 3d bc 20 e2 8c 98 - -There's more. The `%q` (quoted) verb will escape any non-printable -byte sequences in a string so the output is unambiguous. - -.code strings/basic.go /percent q/,/percent q/ - -This technique is handy when much of the string is -intelligible as text but there are peculiarities to root out; it produces: - - "\xbd\xb2=\xbc ⌘" - -If we squint at that, we can see that buried in the noise is one ASCII equals sign, -along with a regular space, and at the end appears the well-known Swedish "Place of Interest" -symbol. -That symbol has Unicode value U+2318, encoded as UTF-8 by the bytes -after the space (hex value `20`): `e2` `8c` `98`. - -If we are unfamiliar or confused by strange values in the string, -we can use the "plus" flag to the `%q` verb. This flag causes the output to escape -not only non-printable sequences, but also any non-ASCII bytes, all -while interpreting UTF-8. -The result is that it exposes the Unicode values of properly formatted UTF-8 -that represents non-ASCII data in the string: - -.code strings/basic.go /percent plus q/,/percent plus q/ - -With that format, the Unicode value of the Swedish symbol shows up as a -`\u` escape: - - "\xbd\xb2=\xbc \u2318" - -These printing techiques are good to know when debugging -the contents of strings, and will be handy in the discussion that follows. -It's worth pointing out as well that all these methods behave exactly the -same for byte slices as they do for strings. - -Here's the full set of printing options we've listed, presented as -a complete program you can run (and edit) right in the browser: - -.play -edit strings/basic.go /package/,/^}/ - -[Exercise: Modify the examples above to use a slice of bytes -instead of a string. Hint: Use a conversion to create the slice.] - -[Exercise: Loop over the string using the `%q` format on each byte. -What does the output tell you?] - -* UTF-8 and string literals - -As we saw, indexing a string yields its bytes, not its characters: a string is just a -bunch of bytes. -That means that when we store a character value in a string, -we store its byte-at-a-time representation. -Let's look at a more controlled example to see how that happens. - -Here's a simple program that prints a string constant with a single character -three different ways, once as a plain string, once as an ASCII-only quoted -string, and once as individual bytes in hexadecimal. -To avoid any confusion, we create a "raw string", enclosed by back quotes, -so it can contain only literal text. (Regular strings, enclosed by double -quotes, can contain escape sequences as we showed above.) - -.play -edit strings/utf8.go /^func/,/^}/ - -The output is: - - plain string: ⌘ - quoted string: "\u2318" - hex bytes: e2 8c 98 - -which reminds us that the Unicode character value U+2318, the "Place -of Interest" symbol ⌘, is represented by the bytes `e2` `8c` `98`, and -that those bytes are the UTF-8 encoding of the hexadecimal -value 2318. - -It may be obvious or it may be subtle, depending on your familiarity with -UTF-8, but it's worth taking a moment to explain how the UTF-8 representation -of the string was created. -The simple fact is: it was created when the source code was written. - -Source code in Go is _defined_ to be UTF-8 text; no other representation is -allowed. That implies that when, in the source code, we write the text - - `⌘` - -the text editor used to create the program places the UTF-8 encoding -of the symbol ⌘ into the source text. -When we print out the hexadecimal bytes, we're just dumping the -data the editor placed in the file. - -In short, Go source code is UTF-8, so -_the_source_code_for_the_string_literal_is_UTF-8_text_. -If that string literal contains no escape sequences, which a raw -string cannot, the constructed string will hold exactly the -source text between the quotes. -Thus by definition and -by construction the raw string will always contain a valid UTF-8 -representation of its contents. -Similarly, unless it contains UTF-8-breaking escapes like those -from the previous section, a regular string literal will also always -contain valid UTF-8. - -Some people think Go strings are always UTF-8, but they -are not: only string literals are UTF-8. -As we showed in the previous section, string _values_ can contain arbitrary -bytes; -as we showed in this one, string _literals_ always contain UTF-8 text -as long as they have no byte-level escapes. - -To summarize, strings can contain arbitrary bytes, but when constructed -from string literals, those bytes are (almost always) UTF-8. - -* Code points, characters, and runes - -We've been very careful so far in how we use the words "byte" and "character". -That's partly because strings hold bytes, and partly because the idea of "character" -is a little hard to define. -The Unicode standard uses the term "code point" to refer to the item represented -by a single value. -The code point U+2318, with hexadecimal value 2318, represents the symbol ⌘. -(For lots more information about that code point, see -[[http://unicode.org/cldr/utility/character.jsp?a=2318][its Unicode page]].) - -To pick a more prosaic example, the Unicode code point U+0061 is the lower -case Latin letter 'A': a. - -But what about the lower case grave-accented letter 'A', à? -That's a character, and it's also a code point (U+00E0), but it has other -representations. -For example we can use the "combining" grave accent code point, U+0300, -and attach it to the lower case letter a, U+0061, to create the same character à. -In general, a character may be represented by a number of different -sequences of code points, and therefore different sequences of UTF-8 bytes. - -The concept of character in computing is therefore ambiguous, or at least -confusing, so we use it with care. -To make things dependable, there are _normalization_ techniques that guarantee that -a given character is always represented by the same code points, but that -subject takes us too far off the topic for now. -A later blog post will explain how the Go libraries address normalization. - -"Code point" is a bit of a mouthful, so Go introduces a shorter term for the -concept: _rune_. -The term appears in the libraries and source code, and means exactly -the same as "code point", with one interesting addition. - -The Go language defines the word `rune` as an alias for the type `int32`, so -programs can be clear when an integer value represents a code point. -Moreover, what you might think of as a character constant is called a -_rune_constant_ in Go. -The type and value of the expression - - '⌘' - -is `rune` with integer value `0x2318`. - -To summarize, here are the salient points: - -- Go source code is always UTF-8. -- A string holds arbitrary bytes. -- A string literal, absent byte-level escapes, always holds valid UTF-8 sequences. -- Those sequences represent Unicode code points, called runes. -- No guarantee is made in Go that characters in strings are normalized. - - -* Range loops - -Besides the axiomatic detail that Go source code is UTF-8, -there's really only one way that Go treats UTF-8 specially, and that is when using -a `for` `range` loop on a string. - -We've seen what happens with a regular `for` loop. -A `for` `range` loop, by contrast, decodes one UTF-8-encoded rune on each -iteration. -Each time around the loop, the index of the loop is the starting position of the -current rune, measured in bytes, and the code point is its value. -Here's an example using yet another handy `Printf` format, `%#U`, which shows -the code point's Unicode value and its printed representation: - -.play -edit strings/range.go /const/,/}/ - -The output shows how each code point occupies multiple bytes: - - U+65E5 '日' starts at byte position 0 - U+672C '本' starts at byte position 3 - U+8A9E '語' starts at byte position 6 - -[Exercise: Put an invalid UTF-8 byte sequence into the string. (How?) -What happens to the iterations of the loop?] - -* Libraries - -Go's standard library provides strong support for interpreting UTF-8 text. -If a `for` `range` loop isn't sufficient for your purposes, -chances are the facility you need is provided by a package in the library. - -The most important such package is -[[http://golang.org/pkg/unicode/utf8/][`unicode/utf8`]], -which contains -helper routines to validate, disassemble, and reassemble UTF-8 strings. -Here is a program equivalent to the `for` `range` example above, -but using the `DecodeRuneInString` function from that package to -do the work. -The return values from the function are the rune and its width in -UTF-8-encoded bytes. - -.play -edit strings/encoding.go /const/,/}/ - -Run it to see that it performs the same. -The `for` `range` loop and `DecodeRuneInString` are defined to produce -exactly the same iteration sequence. - -Look at the -[[http://golang.org/pkg/unicode/utf8/][documentation]] -for the `unicode/utf8` package to see what -other facilities it provides. - -* Conclusion - -To answer the question posed at the beginning: Strings are built from bytes -so indexing them yields bytes, not characters. -A string might not even hold characters. -In fact, the definition of "character" is ambiguous and it would -be a mistake to try to resolve the ambiguity by defining that strings are made -of characters. - -There's much more to say about Unicode, UTF-8, and the world of multilingual -text processing, but it can wait for another post. -For now, we hope you have a better understanding of how Go strings behave -and that, although they may contain arbitrary bytes, UTF-8 is a central part -of their design. diff --git a/content/strings/basic.go b/content/strings/basic.go deleted file mode 100644 index 886e294b..00000000 --- a/content/strings/basic.go +++ /dev/null @@ -1,48 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "fmt" - -func main() { - // const string OMIT - const sample = "\xbd\xb2\x3d\xbc\x20\xe2\x8c\x98" - // const string OMIT - - fmt.Println("Println:") - // println OMIT - fmt.Println(sample) - // println OMIT - - fmt.Println("Byte loop:") - // byte loop OMIT - for i := 0; i < len(sample); i++ { - fmt.Printf("%x ", sample[i]) - } - // byte loop OMIT - fmt.Printf("\n") - - fmt.Println("Printf with %x:") - // percent x OMIT - fmt.Printf("%x\n", sample) - // percent x OMIT - - fmt.Println("Printf with % x:") - // percent space x OMIT - fmt.Printf("% x\n", sample) - // percent space x OMIT - - fmt.Println("Printf with %q:") - // percent q OMIT - fmt.Printf("%q\n", sample) - // percent q OMIT - - fmt.Println("Printf with %+q:") - // percent plus q OMIT - fmt.Printf("%+q\n", sample) - // percent plus q OMIT -} diff --git a/content/strings/encoding.go b/content/strings/encoding.go deleted file mode 100644 index 729b0e50..00000000 --- a/content/strings/encoding.go +++ /dev/null @@ -1,21 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "unicode/utf8" -) - -func main() { - const nihongo = "日本語" - for i, w := 0, 0; i < len(nihongo); i += w { - runeValue, width := utf8.DecodeRuneInString(nihongo[i:]) - fmt.Printf("%#U starts at byte position %d\n", runeValue, i) - w = width - } -} diff --git a/content/strings/range.go b/content/strings/range.go deleted file mode 100644 index e0039fb1..00000000 --- a/content/strings/range.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "fmt" - -func main() { - const nihongo = "日本語" - for index, runeValue := range nihongo { - fmt.Printf("%#U starts at byte position %d\n", runeValue, index) - } -} diff --git a/content/strings/utf8.go b/content/strings/utf8.go deleted file mode 100644 index 491a4462..00000000 --- a/content/strings/utf8.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build OMIT - -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "fmt" - -func main() { - const placeOfInterest = `⌘` - - fmt.Printf("plain string: ") - fmt.Printf("%s", placeOfInterest) - fmt.Printf("\n") - - fmt.Printf("quoted string: ") - fmt.Printf("%+q", placeOfInterest) - fmt.Printf("\n") - - fmt.Printf("hex bytes: ") - for i := 0; i < len(placeOfInterest); i++ { - fmt.Printf("%x ", placeOfInterest[i]) - } - fmt.Printf("\n") -} diff --git a/content/the-app-engine-sdk-and-workspaces-gopath.article b/content/the-app-engine-sdk-and-workspaces-gopath.article deleted file mode 100644 index 873883aa..00000000 --- a/content/the-app-engine-sdk-and-workspaces-gopath.article +++ /dev/null @@ -1,104 +0,0 @@ -The App Engine SDK and workspaces (GOPATH) -9 Jan 2013 -Tags: appengine, tools, gopath - -Andrew Gerrand - -* Introduction - -When we released Go 1 we introduced the [[http://golang.org/cmd/go/][go tool]] and, with it, the concept of workspaces. Workspaces (specified by the GOPATH environment variable) are a convention for organizing code that simplifies fetching, building, and installing Go packages. If you're not familiar with workspaces, please read [[http://golang.org/doc/code.html][this article]] or watch [[http://www.youtube.com/watch?v=XCsL89YtqCs][this screencast]] before reading on. - -Until recently, the tools in the App Engine SDK were not aware of workspaces. Without workspaces the "[[http://golang.org/cmd/go/#hdr-Download_and_install_packages_and_dependencies][go get]]" command cannot function, and so app authors had to install and update their app dependencies manually. It was a pain. - -This has all changed with version 1.7.4 of the App Engine SDK. The [[https://developers.google.com/appengine/docs/go/tools/devserver][dev_appserver]] and [[https://developers.google.com/appengine/docs/go/tools/uploadinganapp][appcfg]] tools are now workspace-aware. When running locally or uploading an app, these tools now search for dependencies in the workspaces specified by the GOPATH environment variable. This means you can now use "go get" while building App Engine apps, and switch between normal Go programs and App Engine apps without changing your environment or habits. - -For example, let's say you want to build an app that uses OAuth 2.0 to authenticate with a remote service. A popular OAuth 2.0 library for Go is the [[http://code.google.com/p/goauth2][goauth2]] package, which you can install to your workspace with this command: - - go get code.google.com/p/goauth2/oauth - -When writing your App Engine app, import the oauth package just as you would in a regular Go program: - - import "code.google.com/p/goauth2/oauth" - -Now, whether running your app with the dev_appserver or deploying it with appcfg, the tools will find the oauth package in your workspace. It just works. - -* Hybrid stand-alone/App Engine apps - -The Go App Engine SDK builds on Go's standard [[http://golang.org/pkg/net/http/][net/http]] package to serve web requests and, as a result, many Go web servers can be run on App Engine with only a few changes. For example, [[http://golang.org/cmd/godoc/][godoc]] is included in the Go distribution as a stand-alone program, but it can also run as an App Engine app (godoc serves [[http://golang.org/][golang.org]] from App Engine). - -But wouldn't it be nice if you could write a program that is both a stand-alone web server and an App Engine app? By using [[http://golang.org/pkg/go/build/#hdr-Build_Constraints][build constraints]], you can. - -Build constraints are line comments that determine whether a file should be included in a package. They are most often used in code that handles a variety of operating systems or processor architectures. For instance, the [[http://golang.org/pkg/path/filepath/][path/filepath]] package includes the file [[http://golang.org/src/pkg/path/filepath/symlink.go][symlink.go]], which specifies a build constraint to ensure that it is not built on Windows systems (which do not have symbolic links): - - // +build !windows - -The App Engine SDK introduces a new build constraint term: "appengine". Files that specify - - // +build appengine - -will be built by the App Engine SDK and ignored by the go tool. Conversely, files that specify - - // +build !appengine - -are ignored by the App Engine SDK, while the go tool will happily build them. - -The [[http://code.google.com/p/goprotobuf/][goprotobuf]] library uses this mechanism to provide two implementations of a key part of its encode/decode machinery: [[http://code.google.com/p/goprotobuf/source/browse/proto/pointer_unsafe.go][pointer_unsafe.go]] is the faster version that cannot be used on App Engine because it uses the [[http://golang.org/pkg/unsafe/][unsafe package]], while [[http://code.google.com/p/goprotobuf/source/browse/proto/pointer_reflect.go][pointer_reflect.go]] is a slower version that avoids unsafe by using the [[http://golang.org/pkg/reflect/][reflect package]] instead. - -Let's take a simple Go web server and turn it into a hybrid app. This is main.go: - - package main - - import ( - "fmt" - "net/http" - ) - - func main() { - http.HandleFunc("/", handler) - http.ListenAndServe("localhost:8080", nil) - } - - func handler(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "Hello!") - } - -Build this with the go tool and you'll get a stand-alone web server executable. - -The App Engine infrastructure provides its own main function that runs its equivalent to ListenAndServe. To convert main.go to an App Engine app, drop the call to ListenAndServe and register the handler in an init function (which runs before main). This is app.go: - - package main - - import ( - "fmt" - "net/http" - ) - - func init() { - http.HandleFunc("/", handler) - } - - func handler(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "Hello!") - } - -To make this a hybrid app, we need to split it into an App Engine-specific part, an stand-alone binary-specific part, and the parts common to both versions. In this case, there is no App Engine-specific part, so we split it into just two files: - -app.go specifies and registers the handler function. It is identical to the code listing above, and requires no build constraints as it should be included in all versions of the program. - -main.go runs the web server. It includes the "!appengine" build constraint, as it must only included when building the stand-alone binary. - - // +build !appengine - - package main - - import "net/http" - - func main() { - http.ListenAndServe("localhost:8080", nil) - } - -To see a more complex hybrid app, take a look at the [[http://code.google.com/p/go/source/browse/?repo=talks#hg%2Fpresent][present tool]]. - -* Conclusions - -We hope these changes will make it easier to work on apps with external dependencies, and to maintain code bases that contain both stand-alone programs and App Engine apps. diff --git a/content/the-path-to-go-1.article b/content/the-path-to-go-1.article deleted file mode 100644 index a2879f5b..00000000 --- a/content/the-path-to-go-1.article +++ /dev/null @@ -1,15 +0,0 @@ -The path to Go 1 -14 Mar 2013 -Tags: talk, video, go1 - -Andrew Gerrand - -* Introduction - -In July 2012, Rob Pike and I presented a talk at OSCON titled _The_path_to_Go_1_. In it we explain how Go 1 came to be, and outline the process by which Go was refined and stabilized to become the clean, consistent programming environment that it is today. We present the major highlights of the release and discuss the details behind some specific libraries and tools. - -.iframe http://www.youtube.com/embed/bj9T2c2Xk_s 309 550 - -The slides for the talk are [[http://talks.golang.org/2012/go1.slide][available here]]. - -It's almost a year since we cut Go 1.0 and we are now busy preparing Go 1.1. The release will include performance improvements to the gc compiler, garbage collector, and goroutine scheduler, some standard library additions, and many bug fixes and other improvements. Stay tuned, as we hope to release Go 1.1 in the coming weeks. diff --git a/content/third-party-libraries-goprotobuf-and.article b/content/third-party-libraries-goprotobuf-and.article deleted file mode 100644 index 70791b88..00000000 --- a/content/third-party-libraries-goprotobuf-and.article +++ /dev/null @@ -1,26 +0,0 @@ -Third-party libraries: goprotobuf and beyond -20 Apr 2010 -Tags: protobuf, community - -Andrew Gerrand - -* Introduction - -On March 24, Rob Pike announced [[http://code.google.com/p/goprotobuf/][goprotobuf]], the Go bindings of Google's data interchange format [[http://code.google.com/apis/protocolbuffers/docs/overview.html][Protocol Buffers]], called protobufs for short. With this announcement, Go joins C++, Java, and Python as languages providing official protobuf implementations. This marks an important milestone in enabling the interoperability between existing systems and those built in Go. - -The goprotobuf project consists of two parts: a 'protocol compiler plugin' that generates Go source files that, once compiled, can access and manage protocol buffers; and a Go package that implements run-time support for encoding (marshaling), decoding (unmarshaling), and accessing protocol buffers. - -To use goprotobuf, you first need to have both Go and [[http://code.google.com/p/protobuf/][protobuf]] installed. You can then install the 'proto' package with [[http://golang.org/cmd/goinstall/][goinstall]]: - - goinstall goprotobuf.googlecode.com/hg/proto - -And then install the protobuf compiler plugin: - - cd $GOROOT/src/pkg/goprotobuf.googlecode.com/hg/compiler - make install - -For more detail see the project's [[http://code.google.com/p/goprotobuf/source/browse/README][README]] file. - -This is one of a growing list of third-party [[http://godashboard.appspot.com/package][Go projects]]. Since the announcement of goprotobuf, the X Go bindings have been spun off from the standard library to the [[http://code.google.com/p/x-go-binding/][x-go-binding]] project, and work has begun on a [[http://www.freetype.org/][Freetype]] port, [[http://code.google.com/p/freetype-go/][freetype-go]]. Other popular third-party projects include the lightweight web framework [[http://github.com/hoisie/web.go][web.go]], and the Go GTK bindings [[http://github.com/mattn/go-gtk][gtk-go]]. - -We wish to encourage the development of other useful packages by the open source community. If you're working on something, don't keep it to yourself - let us know through our mailing list [[http://groups.google.com/group/golang-nuts][golang-nuts]]. diff --git a/content/two-go-talks-lexical-scanning-in-go-and.article b/content/two-go-talks-lexical-scanning-in-go-and.article deleted file mode 100644 index 74696515..00000000 --- a/content/two-go-talks-lexical-scanning-in-go-and.article +++ /dev/null @@ -1,21 +0,0 @@ -Two Go Talks: "Lexical Scanning in Go" and "Cuddle: an App Engine Demo" -1 Sep 2011 -Tags: appengine, lexer, talk, video - -Andrew Gerrand - -* Introduction - -On Tuesday night Rob Pike and Andrew Gerrand each presented at the [[http://www.sydney-gtug.org/][Sydney Google Technology User Group]]. - -Rob's talk, "[[http://www.youtube.com/watch?v=HxaD_trXwRE][Lexical Scanning in Go]]", discusses the design of a particularly interesting and idiomatic piece of Go code, the lexer component of the new [[http://golang.org/pkg/exp/template/][template package.]] - -.iframe http://www.youtube.com/embed/HxaD_trXwRE 345 560 - -The slides are [[http://cuddle.googlecode.com/hg/talk/lex.html][available here]]. The new template package is available as [[http://golang.org/pkg/exp/template/][exp/template]] in Go release r59. In a future release it will replace the old template package. - -Andrew's talk, "[[http://www.youtube.com/watch?v=HQtLRqqB-Kk][Cuddle: an App Engine Demo]]", describes the construction of a simple real-time chat application that uses App Engine's [[http://code.google.com/appengine/docs/go/datastore/overview.html][Datastore]], [[http://code.google.com/appengine/docs/go/channel/overview.html][Channel]], and [[http://code.google.com/appengine/docs/go/datastore/memcache.html][Memcache]] APIs. It also includes a question and answer session that covers [[http://code.google.com/appengine/docs/go/gettingstarted/][Go for App Engine]] and Go more generally. - -.iframe http://www.youtube.com/embed/HQtLRqqB-Kk 345 560 - -The slides are [[http://cuddle.googlecode.com/hg/talk/index.html][available here]]. The code is available at the [[http://code.google.com/p/cuddle/][cuddle Google Code project]]. diff --git a/content/two-recent-go-articles.article b/content/two-recent-go-articles.article deleted file mode 100644 index 63aeb018..00000000 --- a/content/two-recent-go-articles.article +++ /dev/null @@ -1,46 +0,0 @@ -Two recent Go articles -6 Mar 2013 -Tags: google, talk, ethos - -Andrew Gerrand - -* Introduction - -In today's blog post I'd like to highlight a couple of recent articles about Go. - -* Go at Google - -In October last year, Rob Pike presented a keynote at the ACM [[http://splashcon.org/2012/][SPLASH]] conference in Tucson. The talk, titled [[http://talks.golang.org/2012/splash.slide][Go at Google]], was a comprehensive discussion of the motivations behind Go. Rob later expanded on his talk to produce an essay titled [[http://talks.golang.org/2012/splash.article][Go at Google: Language Design in the Service of Software Engineering]]. Here is the abstract: - - The Go programming language was conceived in late 2007 as an - answer to some of the problems we were seeing developing - software infrastructure at Google. The computing landscape - today is almost unrelated to the environment in which the - languages being used, mostly C++, Java, and Python, had been - created. The problems introduced by multicore processors, - networked systems, massive computation clusters, and the web - programming model were being worked around rather than - addressed head-on. Moreover, the scale has changed: today's - server programs comprise tens of millions of lines of code, - are worked on by hundreds or even thousands of programmers, - and are updated literally every day. To make matters worse, - build times, even on large compilation clusters, have - stretched to many minutes, even hours. - - Go was designed and developed to make working in this - environment more productive. Besides its better-known - aspects such as built-in concurrency and garbage collection, - Go's design considerations include rigorous dependency - management, the adaptability of software architecture as - systems grow, and robustness across the boundaries between - components. - -This article explains how these issues were addressed while building an efficient, compiled programming language that feels lightweight and pleasant. Examples and explanations will be taken from the real-world problems faced at Google. - -If you have wondered about the design decisions behind Go, you may find your questions answered by [[http://talks.golang.org/2012/splash.article][the essay]]. It is recommended reading for both new and experienced Go programmers. - -* Go at the Google Developers Academy - -At Google I/O 2012 the Google Developers team [[http://googledevelopers.blogspot.com.au/2012/06/google-launches-new-developer-education.html][launched]] the [[https://developers.google.com/academy/][Google Developers Academy]], a program that provides training materials on Google technologies. Go is one of those technologies and we're pleased to announce the first GDA article featuring Go front and center: - -[[https://developers.google.com/appengine/training/go-plus-appengine/][Getting Started with Go, App Engine and Google+ API]] is an introduction to writing web applications in Go. It demonstrates how to build and deploy App Engine applications and make calls to the Google+ API using the Google APIs Go Client. This is a great entry point for Go programmers eager to get started with Google's developer ecosystem. diff --git a/content/two-recent-go-talks.article b/content/two-recent-go-talks.article deleted file mode 100644 index edcb8680..00000000 --- a/content/two-recent-go-talks.article +++ /dev/null @@ -1,29 +0,0 @@ -Two recent Go talks -2 Jan 2013 -Tags: talk, video, ethos - -Andrew Gerrand - -* Introduction - -Late last year I wrote a couple of Go talks and presented them at [[http://thestrangeloop.com/][Strange Loop]], [[http://oredev.com][Øredev]], and various other venues. The talks are designed to give insight into the practice of Go programming, each describing the construction of a real program and demonstrating the power and depth of the Go language and its libraries and tools. - -The following videos are, in my opinion, the best recordings of these talks. - -* Go: a simple programming environment - -Go is a general-purpose language that bridges the gap between efficient statically typed languages and productive dynamic language. But it’s not just the language that makes Go special – Go has broad and consistent standard libraries and powerful but simple tools. - -This talk gives an introduction to Go, followed by a tour of some real programs that demonstrate the power, scope, and simplicity of the Go programming environment. - -.iframe http://player.vimeo.com/video/53221558?badge=0 281 500 - -See the [[http://talks.golang.org/2012/simple.slide][slide deck]] (use the left and right arrows to navigate). - -* Go: code that grows with grace - -One of Go's key design goals is code adaptability; that it should be easy to take a simple design and build upon it in a clean and natural way. In this talk I describe a simple "chat roulette" server that matches pairs of incoming TCP connections, and then use Go's concurrency mechanisms, interfaces, and standard library to extend it with a web interface and other features. While the function of the program changes dramatically, Go's flexibility preserves the original design as it grows. - -.iframe http://player.vimeo.com/video/53221560?badge=0 281 500 - -See the [[http://talks.golang.org/2012/chat.slide][slide deck]] (use the left and right arrows to navigate). diff --git a/content/upcoming-google-io-go-events.article b/content/upcoming-google-io-go-events.article deleted file mode 100644 index 1641eca8..00000000 --- a/content/upcoming-google-io-go-events.article +++ /dev/null @@ -1,16 +0,0 @@ -Upcoming Google I/O Go Events -12 May 2010 - -Andrew Gerrand - -* Introduction - -[[http://code.google.com/events/io/2010/][Google I/O 2010]] is happening next week at the Moscone Centre in San Francisco. Those of you with tickets will be able to catch some of the Go team both at I/O and at [[http://code.google.com/events/io/2010/bootcamp.html][Bootcamp]]. In reverse-chronological order: - -Rob Pike and Russ Cox will be presenting a [[http://code.google.com/events/io/2010/sessions/go-programming.html][Go Programming]] talk on Thursday at 10.15am. This session takes a detailed look at how Go differs from other languages in a practical sense. Through a series of examples, they will demonstrate various features of Go and the ways in which they affect program design. - -Several members of the Go team will be at the Go cube during [[http://code.google.com/events/io/2010/officehours.html][Office Hours]] on Wednesday between 12pm and 2:30pm. Come by to have your Go questions answered by the experts. - -At Bootcamp on Tuesday at 4.15pm, Andrew Gerrand will be giving an introductory talk about Go. The session will give an overview of the problems that motivated us to build a new language, and the ways in which Go addresses those problems. - -If you're coming to I/O, we look forward to seeing you there! diff --git a/content/writing-scalable-app-engine.article b/content/writing-scalable-app-engine.article deleted file mode 100644 index d260f35b..00000000 --- a/content/writing-scalable-app-engine.article +++ /dev/null @@ -1,18 +0,0 @@ -Writing scalable App Engine applications -1 Nov 2011 -Tags: appengine, optimization - -David Symonds - -* Introduction - -Back in May, we [[http://blog.golang.org/2011/05/go-and-google-app-engine.html][announced]] the Go runtime for App Engine. Since then, we've opened it up for everyone to use, added many new APIs, and improved performance. We have been thrilled by all the interesting ways that people are using Go on App Engine. -One of the key benefits of the Go runtime, apart from working in a fantastic language, is that it has high performance. Go applications compile to native code, with no interpreter or virtual machine getting between your program and the machine. - -Making your web application fast is important because it is well known that a web site's latency has a measurable impact on user happiness, and [[http://googlewebmastercentral.blogspot.com/2010/04/using-site-speed-in-web-search-ranking.html][Google web search uses it as a ranking factor]]. Also announced in May was that App Engine would be [[http://googleappengine.blogspot.com/2011/05/year-ahead-for-google-app-engine.html][leaving its Preview status]] and transitioning to a [[http://www.google.com/enterprise/cloud/appengine/pricing.html][new pricing model]], providing another reason to write efficient App Engine applications. - -To make it easier for Go developers using App Engine to write highly efficient, scalable applications, we recently updated some existing App Engine articles to include snippets of Go source code and to link to relevant Go documentation. - -- [[http://code.google.com/appengine/articles/scaling/overview.html][Best practices for writing scalable applications]] - -- [[http://code.google.com/appengine/articles/managing-resources.html][Managing Your App's Resource Usage]] diff --git a/static/favicon.ico b/static/favicon.ico deleted file mode 100644 index d287722d..00000000 Binary files a/static/favicon.ico and /dev/null differ diff --git a/support/racy/racy.go b/support/racy/racy.go deleted file mode 100644 index e23ba9ba..00000000 --- a/support/racy/racy.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine - -// This program demonstrates a race condition. -// To observe the race with the race detector, build with -race. -package main - -import "fmt" - -func main() { - done := make(chan bool) - m := make(map[string]string) - m["name"] = "world" - go func() { - m["name"] = "data race" - done <- true - }() - fmt.Println("Hello,", m["name"]) - <-done -} diff --git a/template/article.tmpl b/template/article.tmpl deleted file mode 100644 index 2bb3b7b4..00000000 --- a/template/article.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -{{/* This file is combined with the root.tmpl to display a single article. */}} - -{{define "title"}}{{.Doc.Title}} - The Go Blog{{end}} -{{define "content"}} - {{template "doc" .Doc}} - {{with .Doc.Related}} -

Related articles

- - {{end}} -{{end}} - diff --git a/template/doc.tmpl b/template/doc.tmpl deleted file mode 100644 index cb80af7f..00000000 --- a/template/doc.tmpl +++ /dev/null @@ -1,85 +0,0 @@ -{{/* This doc template is given to the present tool to format articles. */}} - -{{define "root"}} - - {{with .Subtitle}}

{{.}}

{{end}} - {{if .Doc | sectioned}} - {{range .Sections}} - {{elem $.Template .}} - {{end}} - {{else}} - {{with index .Sections 0}} - {{range .Elem}} - {{elem $.Template .}} - {{end}} - {{end}} - {{end}} - -{{end}} - -{{define "TOC"}} -
    - {{range .}} -
  • {{.Title}}
    • - {{with .Sections}}{{template "TOC" .}}{{end}} - {{end}} -
-{{end}} - -{{define "newline"}} -{{/* No automatic line break. Paragraphs are free-form. */}} -{{end}} - -{{define "section"}} -

{{.Title}}

- {{range .Elem}}{{elem $.Template .}}{{end}} -{{end}} - -{{define "list"}} -
    - {{range .Bullet}} -
  • {{style .}}
    • - {{end}} -
-{{end}} - -{{define "text"}} - {{if .Pre}} - 
{{range .Lines}}{{.}}{{end}}
- {{else}} -

- {{range $i, $l := .Lines}}{{if $i}}{{template "newline"}} - {{end}}{{style $l}}{{end}} -

- {{end}} -{{end}} - -{{define "code"}} - {{if .Play}} -
{{.Text}}
- {{else}} -
{{.Text}}
- {{end}} -{{end}} - -{{define "image"}} -
- -
-{{end}} - -{{define "caption"}} -
-
{{style .Text}}
-
-{{end}} - -{{define "iframe"}} -
- -
-{{end}} - -{{define "link"}}

{{style .Label}}

{{end}} - -{{define "html"}}{{.HTML}}{{end}} \ No newline at end of file diff --git a/template/home.tmpl b/template/home.tmpl deleted file mode 100644 index 13aaef27..00000000 --- a/template/home.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -{{/* This file is combined with the root.tmpl to display the blog home page. */}} - -{{define "title"}}The Go Programming Language Blog{{end}} -{{define "content"}} - {{range .Data}} - {{template "doc" .}} - {{end}} -

See the index for more articles. -{{end}} diff --git a/template/index.tmpl b/template/index.tmpl deleted file mode 100644 index 0d3c0897..00000000 --- a/template/index.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* This file is combined with the root.tmpl to display the blog index. */}} - -{{define "title"}}Article index - The Go Blog{{end}} -{{define "content"}} - -

Article index

- - {{range .Data}} -

- {{.Title}}
- {{.Time.Format "2 January 2006"}}
- {{with .Tags}}{{range .}}{{.}} {{end}}{{end}} -

- {{end}} - -{{end}} diff --git a/template/root.tmpl b/template/root.tmpl deleted file mode 100644 index 2fa81d0b..00000000 --- a/template/root.tmpl +++ /dev/null @@ -1,170 +0,0 @@ -{{/* This template is combined with other templates to render blog pages. */}} - -{{define "root"}} - - - - - - - {{template "title" .}} - - - - - - - - - -
-
- - - -
-

The Go Blog

- {{template "content" .}} -
- - - -
-
- - - - - - - - -{{end}} - -{{define "doc"}} -
-

{{.Title}}

-

{{.Time.Format "2 January 2006"}}

- {{.HTML}} - {{with .Authors}} -

By {{authors .}}

- {{end}} -
-{{end}}