Wednesday, August 19, 2015

Useful English learning resources

Please find below the list of resources that I find useful for English learning.

Free Apps to install:
Free online courses:
Free video lessons:
Free English tests:
Website with free learning materials:
Website to improve your writing/spelling skills:
Other websites:
Podcasts:

Wednesday, April 17, 2013

Database and LDAP backup scripts

It is a common task to make backups of database or LDAP on the server. Here are examples of such scripts for Linux.

For MySQL database backup I use this script
#!/bin/sh
export PATH=$PATH:/bin:/sbin:/usr/bin:/usr/sbin
umask 077

savedays="180"
backupdir="/mnt/backups/db"

now=`date "+%Y%m%d_%H%M"`

if [ ! -d ${backupdir} ] ; then
    echo Creating ${backupdir}
    mkdir -p  ${backupdir}; chmod 700 ${backupdir}; 
fi

mysqldump -hlocalhost -uuser -ppassword --routines database | gzip > ${backupdir}/database_${now}.sql.gz  2>>/var/log/dbbackups.log
find ${backupdir} -name 'database_*' -a -mtime +${savedays} -print -delete
For LDAP backup I use this script:
#!/bin/sh
export PATH=$PATH:/bin:/sbin:/usr/bin:/usr/sbin
umask 077

savedays="180"
backupdir="/mnt/backups/ldap"

now=`date "+%Y%m%d_%H%M"`

if [ ! -d ${backupdir} ] ; then
    echo Creating ${backupdir}
    mkdir -p  ${backupdir}; chmod 700 ${backupdir}; 
fi
        
slapcat > ${backupdir}/ldap.${now}.ldif  2>>/var/log/ldapbackups.log         
find ${backupdir} -name 'ldap_*' -a -mtime +${savedays} -print -delete
These script will save your database and LDAP backups to /mnt/backups/db and /mnt/backups/ldap accordingly. Backup files will be kept for 180 days and then they will be deleted.

To automate backup you can simply place the scripts to the appropriate cron directory (e.g. /etc/cron.daily) and make sure that cron has rights to execute this file and has rights to write to /mnt/backups/db and /mnt/backups/ldap folders.

Friday, April 12, 2013

Getting Search Volume with Google Adwords API (TargetingIdeaService) - new Java library

Google provides an API to get AdWords data, but there is a little amount of examples of its usage. I'll show you simple example how to get demand (search volume) data for specific words using TargetingIdeaService. In previous post I showed how to do it using the old library, in this post we'll use the new library.

Here is the example:
package loader;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import com.google.api.ads.adwords.axis.factory.AdWordsServices;
import com.google.api.ads.adwords.axis.v201302.cm.Language;
import com.google.api.ads.adwords.axis.v201302.cm.Location;
import com.google.api.ads.adwords.axis.v201302.cm.Money;
import com.google.api.ads.adwords.axis.v201302.cm.Paging;
import com.google.api.ads.adwords.axis.v201302.o.Attribute;
import com.google.api.ads.adwords.axis.v201302.o.AttributeType;
import com.google.api.ads.adwords.axis.v201302.o.IdeaType;
import com.google.api.ads.adwords.axis.v201302.o.LanguageSearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.LocationSearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.LongAttribute;
import com.google.api.ads.adwords.axis.v201302.o.MoneyAttribute;
import com.google.api.ads.adwords.axis.v201302.o.RelatedToQuerySearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.RequestType;
import com.google.api.ads.adwords.axis.v201302.o.SearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.StringAttribute;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdea;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaPage;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaSelector;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaServiceInterface;
import com.google.api.ads.adwords.axis.v201302.o.Type_AttributeMapEntry;
import com.google.api.ads.adwords.lib.client.AdWordsSession;
import com.google.api.ads.common.lib.auth.ClientLoginTokens;
import com.google.api.ads.common.lib.conf.ConfigurationLoadException;
import com.google.api.ads.common.lib.exception.ValidationException;
import com.google.api.client.googleapis.auth.clientlogin.ClientLoginResponseException;

public class Adwords {
    public static void main(String[] args) throws ClientLoginResponseException, IOException, ValidationException,
            ConfigurationLoadException {
        String[] locationNames = new String[] { "Paris", "Quebec", "Spain", "Deutschland" };
        String clientLoginToken = new ClientLoginTokens.Builder().forApi(ClientLoginTokens.Api.ADWORDS)
                .fromFile("adwords.properties").build().requestToken();
        AdWordsSession session = new AdWordsSession.Builder().fromFile("adwords.properties")
                .withClientLoginToken(clientLoginToken).build();
        AdWordsServices adWordsServices = new AdWordsServices();
        String[] keywords = getKeywords();

        TargetingIdeaServiceInterface targetingIdeaService = adWordsServices.get(session,
                TargetingIdeaServiceInterface.class);

        TargetingIdeaSelector selector = new TargetingIdeaSelector();

        selector.setRequestType(RequestType.STATS);
        selector.setIdeaType(IdeaType.KEYWORD);

        selector.setRequestedAttributeTypes(new AttributeType[] { AttributeType.KEYWORD_TEXT, AttributeType.SEARCH_VOLUME, AttributeType.AVERAGE_CPC });

        Language language = new Language();
        language.setId(1000L);

        // Countrycodes
        // http://code.google.com/apis/adwords/docs/appendix/countrycodes.html
        Location location = new Location();
        location.setId(2840L);

        RelatedToQuerySearchParameter relatedToQuerySearchParameter = new RelatedToQuerySearchParameter();
        relatedToQuerySearchParameter.setQueries(keywords);

        LocationSearchParameter locationSearchParameter = new LocationSearchParameter();
        locationSearchParameter.setLocations(new Location[] { location });

        LanguageSearchParameter languageSearchParameter = new LanguageSearchParameter();
        languageSearchParameter.setLanguages(new Language[] { language });

        selector.setSearchParameters(new SearchParameter[] { relatedToQuerySearchParameter, locationSearchParameter,
                languageSearchParameter // if not provided locationSearchParameter, languageSearchParameter then result
                                        // is global
        });

        selector.setLocaleCode("US");

        Paging paging = new Paging();
        paging.setStartIndex(0);
        paging.setNumberResults(keywords.length);
        selector.setPaging(paging);

        TargetingIdeaPage page = targetingIdeaService.get(selector);
        if (page.getEntries() != null && page.getEntries().length > 0) {
            for (TargetingIdea targetingIdea : page.getEntries()) {
                Map<AttributeType, Attribute> data = toMap(targetingIdea.getData());
                String kwd = ((StringAttribute) data.get(AttributeType.KEYWORD_TEXT)).getValue();
                Long monthlySearches = ((LongAttribute) data.get(AttributeType.SEARCH_VOLUME)).getValue();
                Money avgCpc = ((MoneyAttribute) data.get(AttributeType.AVERAGE_CPC)).getValue();
                
                System.out.println(kwd + ", " + monthlySearches + ", " + avgCpc.getMicroAmount() / 1000000.0);
            }
        }
    }
    
    public static String[] getKeywords() {
        //Put your keywords here
        return null;
    }
    
    public static Map<AttributeType, Attribute> toMap(Type_AttributeMapEntry[] data) {
        Map<AttributeType, Attribute> result = new HashMap<AttributeType, Attribute>();
        for (Type_AttributeMapEntry entry: data) {
            result.put(entry.getKey(), entry.getValue());
        }
        return result;        
    }
}

Thursday, February 28, 2013

User friendly Git revision numbers

As you know Git keeps the number of the revision in SHA1 and if you need to show the revision number somewhere in the application you'll have to look for some better solutions. In this post I will investigate the possibilities to display pretty revision numbers.

As a first approach I'd like to mention Maven Buildnumber plugin. It is designed to get a unique build number for each time you build your project. Here is the example snippet from pom.xml:
...
<build>
 <plugins>
  <plugin>
   <groupId>org.codehaus.mojo</groupId>
   <artifactId>buildnumber-maven-plugin</artifactId>
   <version>1.0</version>
   <executions>
       <execution>
    <phase>validate</phase>
    <goals>
        <goal>create</goal>
    </goals>
       </execution>
   </executions>
   <configuration>
       <doCheck>false</doCheck>
       <doUpdate>false</doUpdate>
   </configuration>
  </plugin>
 </plugins>
...
</build>
You'll need also to configure the connection to your git repository:
<scm>
 <connection>scm:git:https://YOUR_CONNECTION</connection>
 <url>scm:git:https://YOUR_URL</url>
 <developerConnection>scm:git:https://YOUR_DEV_CONNECTION</developerConnection>
</scm>
Now you can acces ${buildNumber} within your pom.xml. E.g.:
<plugin>
 <groupId>org.apache.maven.plugins</groupId>
 <artifactId>maven-war-plugin</artifactId>
 <version>2.1</version>
 <configuration>
     <webResources>
  ...
     </webResources>
     <archiveClasses>false</archiveClasses>
     <archive>
  <manifest>
      <addClasspath>true</addClasspath>
      <classpathPrefix/>
  </manifest>
  <manifestEntries>
      <Implementation-Revision>${buildNumber}</Implementation-Revision>
      ...
  </manifestEntries>
     </archive>
 </configuration>
</plugin>
But the ${buildNumber} is too long so it would be better to use additional configuration to use only the beginning N characters. So for example if you will set length of ${buildNumber} to 5 you will get not 6b4017bf51257db678c75732d48121a0997dc19e but 6b401. Here is the configuration for pom.xml:
<configuration>
   <shortRevisionLength>5</shortRevisionLength>
</configuration>
To take ${buildNumber} not only within pom.xml you should use Maven Resources plugin filtering or Maven properties plugin with Spring PropertyPlaceholderConfigurer(For details check here).

The second approach for pretty build number is to use git tag and git describe (for details read here). This is the standard solution for the pretty revision number issue, but if you use CI tools such as Jenkins or Hudson I don't recommend giving to such tools write permissions to your repository. But if CI tool does not have write permissions it cannot create tags and git describe will not give you the result that you expected.

The third approach that you can use in Maven if you have configured resource filtering (the command is for Linux, but I think something similar should work for Windows and Mac):
mvn clean install -DbuildNumberEnv=$(git rev-list HEAD | wc -l)
And now you can use ${buildNumberEnv} to get the numeric value of the revision. But be careful with this approach since your local ${buildNumberEnv} may be the same with another developer's one while you'll have different revision list.
And of course you can use some kind of composition of presented approaches.

Sunday, February 17, 2013

Getting Top Search Queries report with Google Webmaster Tools API

Google has a good and free tool that is very useful for search engine optimization analysis. I'm talking about Google Webmaster Tools. It's possible to download the report from Google WMT using API. It's an example how to do it for Top Queries report.

At first you need to download gdata-webmastertools-2.0.jar and gdata-client-1.0.jar. You can take them from http://gdata-java-client.googlecode.com/files/gdata-src.java-1.47.1.zip (there in the lib folder you'll find the jars).

Then you'll need to place them into you PATH. I will use maven. At first I will install these jars:
mvn install:install-file -Dfile=gdata-webmastertools-2.0.jar -DgroupId=com.google.gdata -DartifactId=gdata-webmastertools -Dversion=2.0 -Dpackaging=jar -DgeneratePom=true
mvn install:install-file -Dfile=gdata-client-1.0.jar -DgroupId=com.google.gdata -DartifactId=gdata-client -Dversion=1.0 -Dpackaging=jar -DgeneratePom=true
After this I will add dependencies to my pom.xml:
<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-client</artifactId>
    <version>1.0</version>
</dependency>
<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-webmastertools</artifactId>
    <version>2.0</version>
</dependency>
Also we'll need to parse JSON. I will use Jackson library:
<dependency>
    <groupId>org.codehaus.jackson</groupId>
    <artifactId>jackson-mapper-asl</artifactId>
    <version>1.8.5</version>
</dependency>
And finally we are ready to get the report from Google WMT API. (In this example I also used Apache Commons and log4j but it's not necessary, so you can get rid off those dependencies)
package loader;

import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.gdata.client.webmastertools.WebmasterToolsService;
import com.google.gdata.data.OutOfLineContent;
import com.google.gdata.data.webmastertools.SitesEntry;
import com.google.gdata.data.webmastertools.SitesFeed;
import com.google.gdata.util.*;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import org.apache.commons.lang.time.DateUtils;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;

public class GoogleWMTClient {
    private final static Logger LOGGER = Logger.getLogger(GoogleWMTClient.class);

    private final static ArrayList<String> STATISTIC_TYPE = new ArrayList<String>();
    static {
        STATISTIC_TYPE.add("ALL");
        // STATISTIC_TYPE.add("WEB");
        // STATISTIC_TYPE.add("IMAGE");
        // STATISTIC_TYPE.add("VIDEO");
        // STATISTIC_TYPE.add("MOBILE_SMARTPHONE");
        // STATISTIC_TYPE.add("MOBILE_RESTRICT");
    }
    private final static String GOOGLE_HOST = "www.google.com";
    private final static String DOWNLOAD_LIST_URL_PART = "/webmasters/tools/downloads-list?hl=%s&siteUrl=%s";
    private final static String SITES_FEED_URL_PART = "/webmasters/tools/feeds/sites/";    
    private final static String DATE_FORMAT = "yyyyMMdd";
    private final static String APPLICATION_NAME = "JavaDevTips";


    public static void main(String[] args) {
        pullData(Constants.GWMT_URL, Constants.GWMT_LANGUAGE_CODE, new Date());
    }
    
    public static void pullData(String url, String languageCode, Date endDate) {
        LOGGER.info("Download GWMT data for endDate: " + endDate + " and url: " + url);
        try {
            WebmasterToolsService service = initService(Constants.ADWORDS_USER, Constants.ADWORDS_PASSWORD);
            // used for deletion of newly created SitesEntry
            boolean newEntry = false;
            SitesEntry entry = findSitesEntry(service, url);
            if (entry == null) {
                newEntry = true;
                try {
                    entry = insertSiteEntry(service, url);
                } catch (ServiceForbiddenException ex) {
                    LOGGER.error(ex, ex);
                }
            }
            downloadReports(service, entry, endDate, languageCode);

            if (newEntry) {
                deleteSiteEntry(service, url);
            }
        } catch (ServiceException e) {
            LOGGER.error(e, e);
        } catch (IOException e) {
            LOGGER.error(e, e);
        }
    }

    public static WebmasterToolsService initService(String userName, String password) throws AuthenticationException {
        WebmasterToolsService service = new WebmasterToolsService(APPLICATION_NAME);
        service.setUserCredentials(userName, password);
        return service;
    }

    private static SitesEntry findSitesEntry(WebmasterToolsService service, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        LOGGER.info("Trying to find SitesEntry for " + siteUrl);
        SitesFeed sitesResultFeed = service.getFeed(getGoogleUrl(SITES_FEED_URL_PART), SitesFeed.class);
        for (SitesEntry entry : sitesResultFeed.getEntries()) {
            if (entry.getTitle().getPlainText().equals(siteUrl)) {
                LOGGER.info("SitesEntry is found");
                return entry;
            }
        }
        LOGGER.info("SitesEntry for " + siteUrl + " not found");
        return null;
    }

    private static URL getGoogleUrl(String path) throws MalformedURLException {
        return new URL("https://" + GOOGLE_HOST + path);
    }

    private static String correctSiteUrl(String siteUrl) {
        siteUrl = siteUrl.trim();
        if (!siteUrl.endsWith("/")) {
            siteUrl += "/";
        }
        if (!siteUrl.startsWith("http")) {
            siteUrl = "http://" + siteUrl;
        }
        return siteUrl;
    }

    private static void downloadReports(WebmasterToolsService service, SitesEntry entry, Date endDate,
            String languageCode) throws IOException, ServiceException {
        LOGGER.info("Downloading reports for " + entry.getTitle().getPlainText());
        Date startDate = DateUtils.addDays(endDate, (-1) * Constants.DATA_PERIOD);
        ObjectMapper mapper = new ObjectMapper();
        InputStream inputStream = getQueryInputStream(service, entry, languageCode);
        if (inputStream == null) {
            LOGGER.error("Empty InputStream");
            return;
        }
        Map<String, Object> map = mapper.readValue(inputStream, new TypeReference<Map<String, Object>>() {
        });
        if (map != null) {
            String fileName = null;
            SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);            
            for (String prop : STATISTIC_TYPE) {
                StringBuilder sbPath = new StringBuilder((String) map.get("TOP_QUERIES")).append("&prop=" + prop)
                        .append("&db=" + sdf.format(startDate))
                        .append("&de=" + sdf.format(endDate));

                fileName = "gwmt_" + sdf.format(endDate) + prop + ".csv" ;
                OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(fileName), "UTF-8");
                boolean hasData = downloadData(service, sbPath.toString(), out);
                if (!hasData) {
                    LOGGER.info("File contains no data. Deleting.");
                    new File(fileName).delete(); // if the file contain no data we delete it
                }
                out.close();
            }
        }
    }

    private static boolean downloadData(WebmasterToolsService service, String path, OutputStreamWriter out)
            throws IOException, ServiceException {
        LOGGER.info("Downloading data for " + path);
        String data;
        URL url = getGoogleUrl(path);
        GDataRequest req = service.createRequest(RequestType.QUERY, url, ContentType.TEXT_PLAIN);
        req.execute();
        BufferedReader in = new BufferedReader(new InputStreamReader(req.getResponseStream()));
        if (in.readLine() != null) {
            while ((data = in.readLine()) != null) {
                out.write(data + "\n");
            }
            return true;
        } else {
            return false;
        }
    }

    private static InputStream getQueryInputStream(WebmasterToolsService service, SitesEntry entry, String lang)
            throws IOException, ServiceException {
        URL url = getGoogleUrl(String.format(DOWNLOAD_LIST_URL_PART, lang, entry.getTitle().getPlainText()));
        GDataRequest req = service.createRequest(RequestType.QUERY, url, ContentType.JSON);
        try {
            req.execute();
            return req.getResponseStream();
        } catch (RedirectRequiredException e) {
            LOGGER.error(e, e);
        }
        return null;
    }

    private static SitesEntry insertSiteEntry(WebmasterToolsService myService, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        SitesEntry entry = new SitesEntry();
        OutOfLineContent content = new OutOfLineContent();
        content.setUri(siteUrl);
        entry.setContent(content);
        LOGGER.info("Adding SitesEntry for  " + siteUrl);
        return myService.insert(getGoogleUrl(SITES_FEED_URL_PART), entry);
    }

    private static void deleteSiteEntry(WebmasterToolsService myService, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        String siteId = URLEncoder.encode(siteUrl, "UTF-8");
        URL feedUrl = new URL(getGoogleUrl(SITES_FEED_URL_PART) + siteId);
        SitesEntry entry = myService.getEntry(feedUrl, SitesEntry.class);
        LOGGER.info("Deleting SitesEntry for " + siteUrl);
        entry.delete();
    }
}